You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by gu...@apache.org on 2015/03/18 06:03:25 UTC
svn commit: r1667454 [3/3] - in /hive/trunk/ql/src:
java/org/apache/hadoop/hive/ql/exec/ java/org/apache/hadoop/hive/ql/exec/mr/
java/org/apache/hadoop/hive/ql/exec/spark/
java/org/apache/hadoop/hive/ql/exec/tez/
java/org/apache/hadoop/hive/ql/exec/vec...
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSMBMapJoinOperator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSMBMapJoinOperator.java?rev=1667454&r1=1667453&r2=1667454&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSMBMapJoinOperator.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSMBMapJoinOperator.java Wed Mar 18 05:03:23 2015
@@ -19,9 +19,11 @@
package org.apache.hadoop.hive.ql.exec.vector;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.Future;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -47,8 +49,8 @@ import org.apache.hadoop.hive.serde2.obj
public class VectorSMBMapJoinOperator extends SMBMapJoinOperator implements VectorizationContextRegion {
private static final Log LOG = LogFactory.getLog(
- VectorSMBMapJoinOperator.class.getName());
-
+ VectorSMBMapJoinOperator.class.getName());
+
private static final long serialVersionUID = 1L;
private VectorExpression[] bigTableValueExpressions;
@@ -65,7 +67,7 @@ public class VectorSMBMapJoinOperator ex
// transient.
//---------------------------------------------------------------------------
- private transient VectorizedRowBatch outputBatch;
+ private transient VectorizedRowBatch outputBatch;
private transient VectorizedRowBatchCtx vrbCtx = null;
@@ -78,23 +80,23 @@ public class VectorSMBMapJoinOperator ex
private transient VectorHashKeyWrapper[] keyValues;
private transient SMBJoinKeyEvaluator keyEvaluator;
-
+
private transient VectorExpressionWriter[] valueWriters;
-
+
private interface SMBJoinKeyEvaluator {
List<Object> evaluate(VectorHashKeyWrapper kw) throws HiveException;
-}
+}
public VectorSMBMapJoinOperator() {
super();
}
-
+
public VectorSMBMapJoinOperator(VectorizationContext vContext, OperatorDesc conf)
throws HiveException {
this();
SMBJoinDesc desc = (SMBJoinDesc) conf;
this.conf = desc;
-
+
order = desc.getTagOrder();
numAliases = desc.getExprs().size();
posBigTable = (byte) desc.getPosBigTable();
@@ -118,7 +120,7 @@ public class VectorSMBMapJoinOperator ex
vOutContext = new VectorizationContext(desc.getOutputColumnNames());
vOutContext.setFileKey(vContext.getFileKey() + "/SMB_JOIN_" + desc.getBigTableAlias());
}
-
+
@Override
protected List<Object> smbJoinComputeKeys(Object row, byte alias) throws HiveException {
if (alias == this.posBigTable) {
@@ -127,21 +129,21 @@ public class VectorSMBMapJoinOperator ex
} else {
return super.smbJoinComputeKeys(row, alias);
}
- }
-
+ }
+
@Override
- protected void initializeOp(Configuration hconf) throws HiveException {
- super.initializeOp(hconf);
+ protected Collection<Future<?>> initializeOp(Configuration hconf) throws HiveException {
+ Collection<Future<?>> result = super.initializeOp(hconf);
vrbCtx = new VectorizedRowBatchCtx();
vrbCtx.init(vOutContext.getScratchColumnTypeMap(), (StructObjectInspector) this.outputObjInspector);
-
+
outputBatch = vrbCtx.createVectorizedRowBatch();
-
+
keyWrapperBatch = VectorHashKeyWrapperBatch.compileKeyWrapperBatch(keyExpressions);
-
+
outputVectorAssigners = new HashMap<ObjectInspector, VectorColumnAssign[]>();
-
+
// This key evaluator translates from the vectorized VectorHashKeyWrapper format
// into the row-mode MapJoinKey
keyEvaluator = new SMBJoinKeyEvaluator() {
@@ -163,14 +165,14 @@ public class VectorSMBMapJoinOperator ex
return key;
};
}.init();
-
+
Map<Byte, List<ExprNodeDesc>> valueExpressions = conf.getExprs();
- List<ExprNodeDesc> bigTableExpressions = valueExpressions.get(posBigTable);
-
+ List<ExprNodeDesc> bigTableExpressions = valueExpressions.get(posBigTable);
+
// We're hijacking the big table evaluators and replacing them with our own custom ones
// which are going to return values from the input batch vector expressions
List<ExprNodeEvaluator> vectorNodeEvaluators = new ArrayList<ExprNodeEvaluator>(bigTableExpressions.size());
-
+
VectorExpressionWriterFactory.processVectorExpressions(
bigTableExpressions,
new VectorExpressionWriterFactory.ListOIDClosure() {
@@ -180,7 +182,7 @@ public class VectorSMBMapJoinOperator ex
valueWriters = writers;
joinValuesObjectInspectors[posBigTable] = oids;
}
- });
+ });
for(int i=0; i<bigTableExpressions.size(); ++i) {
ExprNodeDesc desc = bigTableExpressions.get(i);
@@ -213,51 +215,51 @@ public class VectorSMBMapJoinOperator ex
}
// Now replace the old evaluators with our own
joinValues[posBigTable] = vectorNodeEvaluators;
-
+ return result;
}
-
+
@Override
- public void processOp(Object row, int tag) throws HiveException {
+ public void process(Object row, int tag) throws HiveException {
byte alias = (byte) tag;
-
+
if (alias != this.posBigTable) {
- super.processOp(row, tag);
+ super.process(row, tag);
} else {
-
+
VectorizedRowBatch inBatch = (VectorizedRowBatch) row;
-
+
if (null != bigTableFilterExpressions) {
for(VectorExpression ve : bigTableFilterExpressions) {
ve.evaluate(inBatch);
}
}
-
+
if (null != bigTableValueExpressions) {
for(VectorExpression ve : bigTableValueExpressions) {
ve.evaluate(inBatch);
}
}
-
+
keyWrapperBatch.evaluateBatch(inBatch);
keyValues = keyWrapperBatch.getVectorHashKeyWrappers();
-
+
// This implementation of vectorized JOIN is delegating all the work
// to the row-mode implementation by hijacking the big table node evaluators
// and calling the row-mode join processOp for each row in the input batch.
- // Since the JOIN operator is not fully vectorized anyway at the moment
+ // Since the JOIN operator is not fully vectorized anyway at the moment
// (due to the use of row-mode small-tables) this is a reasonable trade-off.
//
for(batchIndex=0; batchIndex < inBatch.size; ++batchIndex ) {
- super.processOp(row, tag);
+ super.process(row, tag);
}
-
+
// Set these two to invalid values so any attempt to use them
// outside the inner loop results in NPE/OutOfBounds errors
batchIndex = -1;
keyValues = null;
}
}
-
+
@Override
public void closeOp(boolean aborted) throws HiveException {
super.closeOp(aborted);
@@ -265,7 +267,7 @@ public class VectorSMBMapJoinOperator ex
flushOutput();
}
}
-
+
@Override
protected void internalForward(Object row, ObjectInspector outputOI) throws HiveException {
Object[] values = (Object[]) row;
@@ -283,11 +285,11 @@ public class VectorSMBMapJoinOperator ex
flushOutput();
}
}
-
+
private void flushOutput() throws HiveException {
forward(outputBatch, null);
outputBatch.reset();
- }
+ }
@Override
public VectorizationContext getOuputVectorizationContext() {
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSelectOperator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSelectOperator.java?rev=1667454&r1=1667453&r2=1667454&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSelectOperator.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSelectOperator.java Wed Mar 18 05:03:23 2015
@@ -19,11 +19,12 @@
package org.apache.hadoop.hive.ql.exec.vector;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.List;
-import java.util.Map;
+import java.util.concurrent.Future;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.ql.exec.SelectOperator;
+import org.apache.hadoop.hive.ql.exec.Operator;
import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpressionWriter;
import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpressionWriterFactory;
@@ -31,13 +32,15 @@ import org.apache.hadoop.hive.ql.metadat
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
import org.apache.hadoop.hive.ql.plan.OperatorDesc;
import org.apache.hadoop.hive.ql.plan.SelectDesc;
+import org.apache.hadoop.hive.ql.plan.api.OperatorType;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
/**
* Select operator implementation.
*/
-public class VectorSelectOperator extends SelectOperator implements VectorizationContextRegion {
+public class VectorSelectOperator extends Operator<SelectDesc> implements
+ VectorizationContextRegion {
private static final long serialVersionUID = 1L;
@@ -62,7 +65,7 @@ public class VectorSelectOperator extend
}
/**
- * Create a new vectorization context to create a new projection, but keep
+ * Create a new vectorization context to create a new projection, but keep
* same output column manager must be inherited to track the scratch the columns.
*/
vOutContext = new VectorizationContext(vContext);
@@ -74,7 +77,7 @@ public class VectorSelectOperator extend
for (int i=0; i < colList.size(); ++i) {
String columnName = this.conf.getOutputColumnNames().get(i);
VectorExpression ve = vExpressions[i];
- vOutContext.addProjectionColumn(columnName,
+ vOutContext.addProjectionColumn(columnName,
ve.getOutputColumn());
}
}
@@ -83,11 +86,11 @@ public class VectorSelectOperator extend
}
@Override
- protected void initializeOp(Configuration hconf) throws HiveException {
+ protected Collection<Future<?>> initializeOp(Configuration hconf) throws HiveException {
+ Collection<Future<?>> result = super.initializeOp(hconf);
// Just forward the row as is
if (conf.isSelStarNoCompute()) {
- initializeChildren(hconf);
- return;
+ return null;
}
List<ObjectInspector> objectInspectors = new ArrayList<ObjectInspector>();
@@ -102,15 +105,15 @@ public class VectorSelectOperator extend
outputObjInspector = ObjectInspectorFactory.getStandardStructObjectInspector(
outputFieldNames, objectInspectors);
- initializeChildren(hconf);
projectedColumns = new int [vExpressions.length];
for (int i = 0; i < projectedColumns.length; i++) {
projectedColumns[i] = vExpressions[i].getOutputColumn();
}
+ return result;
}
@Override
- public void processOp(Object row, int tag) throws HiveException {
+ public void process(Object row, int tag) throws HiveException {
// Just forward the row as is
if (conf.isSelStarNoCompute()) {
@@ -167,4 +170,9 @@ public class VectorSelectOperator extend
public VectorizationContext getOuputVectorizationContext() {
return vOutContext;
}
+
+ @Override
+ public OperatorType getType() {
+ return OperatorType.SELECT;
+ }
}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/merge/MergeFileMapper.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/merge/MergeFileMapper.java?rev=1667454&r1=1667453&r2=1667454&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/merge/MergeFileMapper.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/merge/MergeFileMapper.java Wed Mar 18 05:03:23 2015
@@ -97,7 +97,7 @@ public class MergeFileMapper extends Map
row[0] = key;
row[1] = value;
try {
- mergeOp.processOp(row, 0);
+ mergeOp.process(row, 0);
} catch (HiveException e) {
abort = true;
throw new IOException(e);
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lib/ForwardWalker.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lib/ForwardWalker.java?rev=1667454&r1=1667453&r2=1667454&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lib/ForwardWalker.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/lib/ForwardWalker.java Wed Mar 18 05:03:23 2015
@@ -54,9 +54,6 @@ public class ForwardWalker extends Defau
@SuppressWarnings("unchecked")
protected void addAllParents(Node nd) {
Operator<? extends OperatorDesc> op = (Operator<? extends OperatorDesc>) nd;
- if (op.getParentOperators() == null) {
- return;
- }
getToWalk().removeAll(op.getParentOperators());
getToWalk().addAll(0, op.getParentOperators());
}
@@ -68,6 +65,7 @@ public class ForwardWalker extends Defau
* current operator in the graph
* @throws SemanticException
*/
+ @Override
public void walk(Node nd) throws SemanticException {
if (opStack.empty() || nd != opStack.peek()) {
opStack.push(nd);
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java?rev=1667454&r1=1667453&r2=1667454&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java Wed Mar 18 05:03:23 2015
@@ -273,16 +273,7 @@ public class ConvertJoinMapJoin implemen
}
List<Operator<? extends OperatorDesc>> childOperators = mergeJoinOp.getChildOperators();
- if (childOperators == null) {
- childOperators = new ArrayList<Operator<? extends OperatorDesc>>();
- mergeJoinOp.setChildOperators(childOperators);
- }
-
List<Operator<? extends OperatorDesc>> parentOperators = mergeJoinOp.getParentOperators();
- if (parentOperators == null) {
- parentOperators = new ArrayList<Operator<? extends OperatorDesc>>();
- mergeJoinOp.setParentOperators(parentOperators);
- }
childOperators.clear();
parentOperators.clear();
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java?rev=1667454&r1=1667453&r2=1667454&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java Wed Mar 18 05:03:23 2015
@@ -251,10 +251,8 @@ public class MapJoinProcessor implements
if (!op.opAllowedAfterMapJoin()) {
throw new SemanticException(ErrorMsg.OPERATOR_NOT_ALLOWED_WITH_MAPJOIN.getMsg());
}
- if (op.getChildOperators() != null) {
- for (Operator<? extends OperatorDesc> childOp : op.getChildOperators()) {
- checkChildOperatorType(childOp);
- }
+ for (Operator<? extends OperatorDesc> childOp : op.getChildOperators()) {
+ checkChildOperatorType(childOp);
}
}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/metainfo/annotation/OpTraitsRulesProcFactory.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/metainfo/annotation/OpTraitsRulesProcFactory.java?rev=1667454&r1=1667453&r2=1667454&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/metainfo/annotation/OpTraitsRulesProcFactory.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/metainfo/annotation/OpTraitsRulesProcFactory.java Wed Mar 18 05:03:23 2015
@@ -107,7 +107,7 @@ public class OpTraitsRulesProcFactory {
listBucketCols.add(bucketCols);
int numBuckets = -1;
int numReduceSinks = 1;
- OpTraits parentOpTraits = rs.getParentOperators().get(0).getConf().getOpTraits();
+ OpTraits parentOpTraits = rs.getParentOperators().get(0).getConf().getTraits();
if (parentOpTraits != null) {
numBuckets = parentOpTraits.getNumBuckets();
numReduceSinks += parentOpTraits.getNumReduceSinks();
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/CommonJoinTaskDispatcher.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/CommonJoinTaskDispatcher.java?rev=1667454&r1=1667453&r2=1667454&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/CommonJoinTaskDispatcher.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/CommonJoinTaskDispatcher.java Wed Mar 18 05:03:23 2015
@@ -554,10 +554,6 @@ public class CommonJoinTaskDispatcher ex
return false;
}
- if (op.getChildOperators() == null) {
- return true;
- }
-
for (Operator<? extends OperatorDesc> childOp : op.getChildOperators()) {
if (!checkOperatorOKMapJoinConversion(childOp)) {
return false;
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SplitSparkWorkResolver.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SplitSparkWorkResolver.java?rev=1667454&r1=1667453&r2=1667454&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SplitSparkWorkResolver.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SplitSparkWorkResolver.java Wed Mar 18 05:03:23 2015
@@ -183,7 +183,7 @@ public class SplitSparkWorkResolver impl
private void setStatistics(Operator<? extends OperatorDesc> origin,
Operator<? extends OperatorDesc> clone) {
clone.getConf().setStatistics(origin.getConf().getStatistics());
- clone.getConf().setOpTraits(origin.getConf().getOpTraits());
+ clone.getConf().setTraits(origin.getConf().getTraits());
if (origin.getChildOperators().size() == clone.getChildOperators().size()) {
for (int i = 0; i < clone.getChildOperators().size(); i++) {
setStatistics(origin.getChildOperators().get(i), clone.getChildOperators().get(i));
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/MapReduceCompiler.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/MapReduceCompiler.java?rev=1667454&r1=1667453&r2=1667454&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/MapReduceCompiler.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/MapReduceCompiler.java Wed Mar 18 05:03:23 2015
@@ -152,10 +152,6 @@ public class MapReduceCompiler extends T
topOp.setChildOperators(null);
}
- if (topOp.getChildOperators() == null) {
- return;
- }
-
for (Operator<? extends OperatorDesc> op : topOp.getChildOperators()) {
breakOperatorTree(op);
}
@@ -194,6 +190,7 @@ public class MapReduceCompiler extends T
final Context lCtx = ctx;
PathFilter p = new PathFilter() {
+ @Override
public boolean accept(Path file) {
return !lCtx.isMRTmpFileURI(file.toUri().getPath());
}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/AbstractOperatorDesc.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/AbstractOperatorDesc.java?rev=1667454&r1=1667453&r2=1667454&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/AbstractOperatorDesc.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/AbstractOperatorDesc.java Wed Mar 18 05:03:23 2015
@@ -54,11 +54,11 @@ public class AbstractOperatorDesc implem
this.vectorMode = vm;
}
- public OpTraits getOpTraits() {
+ public OpTraits getTraits() {
return opTraits;
}
- public void setOpTraits(OpTraits opTraits) {
+ public void setTraits(OpTraits opTraits) {
this.opTraits = opTraits;
}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/OperatorDesc.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/OperatorDesc.java?rev=1667454&r1=1667453&r2=1667454&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/OperatorDesc.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/OperatorDesc.java Wed Mar 18 05:03:23 2015
@@ -25,7 +25,7 @@ public interface OperatorDesc extends Se
public Object clone() throws CloneNotSupportedException;
public Statistics getStatistics();
public void setStatistics(Statistics statistics);
- public OpTraits getOpTraits();
- public void setOpTraits(OpTraits opTraits);
+ public OpTraits getTraits();
+ public void setTraits(OpTraits opTraits);
public Map<String, String> getOpProps();
}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/ppd/PredicateTransitivePropagate.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/ppd/PredicateTransitivePropagate.java?rev=1667454&r1=1667453&r2=1667454&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/ppd/PredicateTransitivePropagate.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/ppd/PredicateTransitivePropagate.java Wed Mar 18 05:03:23 2015
@@ -108,8 +108,6 @@ public class PredicateTransitivePropagat
RowSchema parentRS, ExprNodeDesc filterExpr) {
Operator<FilterDesc> filter = OperatorFactory.get(new FilterDesc(filterExpr, false),
new RowSchema(parentRS.getSignature()));
- filter.setParentOperators(new ArrayList<Operator<? extends OperatorDesc>>());
- filter.setChildOperators(new ArrayList<Operator<? extends OperatorDesc>>());
filter.getParentOperators().add(parent);
filter.getChildOperators().add(target);
parent.replaceChild(target, filter);
@@ -224,7 +222,7 @@ public class PredicateTransitivePropagat
private static class Vectors {
- private Set<Integer>[] vector;
+ private final Set<Integer>[] vector;
@SuppressWarnings("unchecked")
public Vectors(int length) {
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/ppd/SyntheticJoinPredicate.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/ppd/SyntheticJoinPredicate.java?rev=1667454&r1=1667453&r2=1667454&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/ppd/SyntheticJoinPredicate.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/ppd/SyntheticJoinPredicate.java Wed Mar 18 05:03:23 2015
@@ -97,8 +97,6 @@ public class SyntheticJoinPredicate impl
RowSchema parentRS, ExprNodeDesc filterExpr) {
Operator<FilterDesc> filter = OperatorFactory.get(new FilterDesc(filterExpr, false),
new RowSchema(parentRS.getSignature()));
- filter.setParentOperators(new ArrayList<Operator<? extends OperatorDesc>>());
- filter.setChildOperators(new ArrayList<Operator<? extends OperatorDesc>>());
filter.getParentOperators().add(parent);
filter.getChildOperators().add(target);
parent.replaceChild(target, filter);
Modified: hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/TestFileSinkOperator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/TestFileSinkOperator.java?rev=1667454&r1=1667453&r2=1667454&view=diff
==============================================================================
--- hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/TestFileSinkOperator.java (original)
+++ hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/TestFileSinkOperator.java Wed Mar 18 05:03:23 2015
@@ -319,7 +319,7 @@ public class TestFileSinkOperator {
}
private void processRows(FileSinkOperator op) throws HiveException {
- for (TFSORow r : rows) op.processOp(r, 0);
+ for (TFSORow r : rows) op.process(r, 0);
op.jobCloseOp(jc, true);
op.close(false);
}
Modified: hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/TestOperators.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/TestOperators.java?rev=1667454&r1=1667453&r2=1667454&view=diff
==============================================================================
--- hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/TestOperators.java (original)
+++ hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/TestOperators.java Wed Mar 18 05:03:23 2015
@@ -236,7 +236,7 @@ public class TestOperators extends TestC
// evaluate on row
for (int i = 0; i < 5; i++) {
- op.processOp(r[i].o, 0);
+ op.process(r[i].o, 0);
}
op.close(false);
Modified: hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorFilterOperator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorFilterOperator.java?rev=1667454&r1=1667453&r2=1667454&view=diff
==============================================================================
--- hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorFilterOperator.java (original)
+++ hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorFilterOperator.java Wed Mar 18 05:03:23 2015
@@ -19,12 +19,11 @@
package org.apache.hadoop.hive.ql.exec.vector;
import java.util.ArrayList;
-import java.util.HashMap;
import java.util.List;
-import java.util.Map;
import junit.framework.Assert;
+import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.exec.vector.expressions.FilterExprAndExpr;
import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FilterLongColEqualDoubleScalar;
@@ -39,6 +38,8 @@ import org.junit.Test;
*/
public class TestVectorFilterOperator {
+ HiveConf hconf = new HiveConf();
+
/**
* Fundamental logic and performance tests for vector filters belong here.
*
@@ -96,6 +97,7 @@ public class TestVectorFilterOperator {
@Test
public void testBasicFilterOperator() throws HiveException {
VectorFilterOperator vfo = getAVectorFilterOperator();
+ vfo.initialize(hconf, null);
VectorExpression ve1 = new FilterLongColGreaterLongColumn(0,1);
VectorExpression ve2 = new FilterLongColEqualDoubleScalar(2, 0);
VectorExpression ve3 = new FilterExprAndExpr();
@@ -124,6 +126,7 @@ public class TestVectorFilterOperator {
@Test
public void testBasicFilterLargeData() throws HiveException {
VectorFilterOperator vfo = getAVectorFilterOperator();
+ vfo.initialize(hconf, null);
VectorExpression ve1 = new FilterLongColGreaterLongColumn(0,1);
VectorExpression ve2 = new FilterLongColEqualDoubleScalar(2, 0);
VectorExpression ve3 = new FilterExprAndExpr();
@@ -136,7 +139,7 @@ public class TestVectorFilterOperator {
VectorizedRowBatch vrg = fdr.getNext();
while (vrg.size > 0) {
- vfo.processOp(vrg, 0);
+ vfo.process(vrg, 0);
vrg = fdr.getNext();
}
long endTime = System.currentTimeMillis();
Modified: hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorGroupByOperator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorGroupByOperator.java?rev=1667454&r1=1667453&r2=1667454&view=diff
==============================================================================
--- hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorGroupByOperator.java (original)
+++ hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorGroupByOperator.java Wed Mar 18 05:03:23 2015
@@ -26,8 +26,6 @@ import static org.junit.Assert.assertTru
import java.lang.management.ManagementFactory;
import java.lang.management.MemoryMXBean;
import java.lang.reflect.Constructor;
-import java.math.BigDecimal;
-import java.math.BigInteger;
import java.sql.Timestamp;
import java.util.ArrayList;
import java.util.Arrays;
@@ -39,6 +37,7 @@ import java.util.Map;
import java.util.Set;
import org.apache.hadoop.hive.common.type.HiveDecimal;
+import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.exec.vector.util.FakeCaptureOutputOperator;
import org.apache.hadoop.hive.ql.exec.vector.util.FakeVectorRowBatchFromConcat;
import org.apache.hadoop.hive.ql.exec.vector.util.FakeVectorRowBatchFromLongIterables;
@@ -70,6 +69,8 @@ import org.junit.Test;
*/
public class TestVectorGroupByOperator {
+ HiveConf hconf = new HiveConf();
+
private static ExprNodeDesc buildColumnDesc(
VectorizationContext ctx,
String column,
@@ -188,7 +189,7 @@ public class TestVectorGroupByOperator {
VectorGroupByOperator vgo = new VectorGroupByOperator(ctx, desc);
FakeCaptureOutputOperator out = FakeCaptureOutputOperator.addCaptureOutputChild(vgo);
- vgo.initialize(null, null);
+ vgo.initialize(hconf, null);
this.outputRowCount = 0;
out.setOutputInspector(new FakeCaptureOutputOperator.OutputInspector() {
@@ -233,7 +234,7 @@ public class TestVectorGroupByOperator {
long countRowsProduced = 0;
for (VectorizedRowBatch unit: data) {
countRowsProduced += 100;
- vgo.processOp(unit, 0);
+ vgo.process(unit, 0);
if (0 < outputRowCount) {
break;
}
@@ -529,7 +530,7 @@ public class TestVectorGroupByOperator {
new String[] {"int", "bigint"},
Arrays.asList(new Object[]{ 1,null, 1, null}),
Arrays.asList(new Object[]{13L,null,7L, 19L})),
- buildHashMap((int)1, 20L, null, 19L));
+ buildHashMap(1, 20L, null, 19L));
}
@Test
@@ -541,7 +542,7 @@ public class TestVectorGroupByOperator {
new String[] {"bigint", "bigint"},
Arrays.asList(new Object[]{ 1,null, 1, null}),
Arrays.asList(new Object[]{13L,null,7L, 19L})),
- buildHashMap((long)1L, 20L, null, 19L));
+ buildHashMap(1L, 20L, null, 19L));
}
@Test
@@ -589,7 +590,7 @@ public class TestVectorGroupByOperator {
new String[] {"double", "bigint"},
Arrays.asList(new Object[]{ 1,null, 1, null}),
Arrays.asList(new Object[]{13L,null,7L, 19L})),
- buildHashMap((double)1.0, 20L, null, 19L));
+ buildHashMap(1.0, 20L, null, 19L));
}
@Test
@@ -794,7 +795,7 @@ public class TestVectorGroupByOperator {
HiveDecimal.create(5),
HiveDecimal.create(7),
HiveDecimal.create(19)}),
- (double) Math.sqrt(30));
+ Math.sqrt(30));
}
@Test
@@ -808,7 +809,7 @@ public class TestVectorGroupByOperator {
HiveDecimal.create(5),
HiveDecimal.create(7),
HiveDecimal.create(19)}),
- (double) Math.sqrt(40));
+ Math.sqrt(40));
}
@Test
@@ -1546,7 +1547,7 @@ public class TestVectorGroupByOperator {
"variance",
2,
Arrays.asList(new Long[]{97L}),
- (double)0.0);
+ 0.0);
}
@Test
@@ -1565,12 +1566,12 @@ public class TestVectorGroupByOperator {
"variance",
2,
Arrays.asList(new Long[]{null,13L, 5L,7L,19L}),
- (double) 30.0);
+ 30.0);
testAggregateLongAggregate(
"variance",
2,
Arrays.asList(new Long[]{13L,null,5L, 7L,19L}),
- (double) 30.0);
+ 30.0);
testAggregateLongAggregate(
"variance",
2,
@@ -1633,7 +1634,7 @@ public class TestVectorGroupByOperator {
"std",
2,
Arrays.asList(new Long[]{13L,5L,7L,19L}),
- (double) Math.sqrt(30));
+ Math.sqrt(30));
}
@Test
@@ -1673,7 +1674,7 @@ public class TestVectorGroupByOperator {
"stddev_samp",
2,
Arrays.asList(new Long[]{13L,5L,7L,19L}),
- (double) Math.sqrt(40));
+ Math.sqrt(40));
}
@Test
@@ -1731,7 +1732,7 @@ public class TestVectorGroupByOperator {
VectorGroupByOperator vgo = new VectorGroupByOperator(ctx, desc);
FakeCaptureOutputOperator out = FakeCaptureOutputOperator.addCaptureOutputChild(vgo);
- vgo.initialize(null, null);
+ vgo.initialize(hconf, null);
out.setOutputInspector(new FakeCaptureOutputOperator.OutputInspector() {
private int rowIndex;
@@ -1801,7 +1802,7 @@ public class TestVectorGroupByOperator {
}.init(aggregateName, expected, keys));
for (VectorizedRowBatch unit: data) {
- vgo.processOp(unit, 0);
+ vgo.process(unit, 0);
}
vgo.close(false);
@@ -1845,7 +1846,7 @@ public class TestVectorGroupByOperator {
VectorGroupByOperator vgo = new VectorGroupByOperator(ctx, desc);
FakeCaptureOutputOperator out = FakeCaptureOutputOperator.addCaptureOutputChild(vgo);
- vgo.initialize(null, null);
+ vgo.initialize(hconf, null);
out.setOutputInspector(new FakeCaptureOutputOperator.OutputInspector() {
private int rowIndex;
@@ -1914,7 +1915,7 @@ public class TestVectorGroupByOperator {
}.init(aggregateName, expected, keys));
for (VectorizedRowBatch unit: data) {
- vgo.processOp(unit, 0);
+ vgo.process(unit, 0);
}
vgo.close(false);
@@ -2079,18 +2080,18 @@ public class TestVectorGroupByOperator {
assertEquals(key, null, arr[0]);
} else if (arr[0] instanceof LongWritable) {
LongWritable lw = (LongWritable) arr[0];
- assertEquals(key, (Long) expected, (Long) lw.get());
+ assertEquals(key, expected, lw.get());
} else if (arr[0] instanceof Text) {
Text tx = (Text) arr[0];
String sbw = tx.toString();
- assertEquals(key, (String) expected, sbw);
+ assertEquals(key, expected, sbw);
} else if (arr[0] instanceof DoubleWritable) {
DoubleWritable dw = (DoubleWritable) arr[0];
- assertEquals (key, (Double) expected, (Double) dw.get());
+ assertEquals (key, expected, dw.get());
} else if (arr[0] instanceof Double) {
- assertEquals (key, (Double) expected, (Double) arr[0]);
+ assertEquals (key, expected, arr[0]);
} else if (arr[0] instanceof Long) {
- assertEquals (key, (Long) expected, (Long) arr[0]);
+ assertEquals (key, expected, arr[0]);
} else if (arr[0] instanceof HiveDecimalWritable) {
HiveDecimalWritable hdw = (HiveDecimalWritable) arr[0];
HiveDecimal hd = hdw.getHiveDecimal();
@@ -2126,10 +2127,10 @@ public class TestVectorGroupByOperator {
if (vals[1] instanceof DoubleWritable) {
DoubleWritable dw = (DoubleWritable) vals[1];
- assertEquals (key, (Double) expected, (Double) (dw.get() / lw.get()));
+ assertEquals (key, expected, dw.get() / lw.get());
} else if (vals[1] instanceof HiveDecimalWritable) {
HiveDecimalWritable hdw = (HiveDecimalWritable) vals[1];
- assertEquals (key, (HiveDecimal) expected, hdw.getHiveDecimal().divide(HiveDecimal.create(lw.get())));
+ assertEquals (key, expected, hdw.getHiveDecimal().divide(HiveDecimal.create(lw.get())));
}
}
}
@@ -2241,10 +2242,10 @@ public class TestVectorGroupByOperator {
VectorGroupByOperator vgo = new VectorGroupByOperator(ctx, desc);
FakeCaptureOutputOperator out = FakeCaptureOutputOperator.addCaptureOutputChild(vgo);
- vgo.initialize(null, null);
+ vgo.initialize(hconf, null);
for (VectorizedRowBatch unit: data) {
- vgo.processOp(unit, 0);
+ vgo.process(unit, 0);
}
vgo.close(false);
@@ -2272,10 +2273,10 @@ public class TestVectorGroupByOperator {
VectorGroupByOperator vgo = new VectorGroupByOperator(ctx, desc);
FakeCaptureOutputOperator out = FakeCaptureOutputOperator.addCaptureOutputChild(vgo);
- vgo.initialize(null, null);
+ vgo.initialize(hconf, null);
for (VectorizedRowBatch unit: data) {
- vgo.processOp(unit, 0);
+ vgo.process(unit, 0);
}
vgo.close(false);
@@ -2303,10 +2304,10 @@ public class TestVectorGroupByOperator {
VectorGroupByOperator vgo = new VectorGroupByOperator(ctx, desc);
FakeCaptureOutputOperator out = FakeCaptureOutputOperator.addCaptureOutputChild(vgo);
- vgo.initialize(null, null);
+ vgo.initialize(hconf, null);
for (VectorizedRowBatch unit: data) {
- vgo.processOp(unit, 0);
+ vgo.process(unit, 0);
}
vgo.close(false);
@@ -2321,35 +2322,34 @@ public class TestVectorGroupByOperator {
}
public void testAggregateDecimalIterable (
- String aggregateName,
- Iterable<VectorizedRowBatch> data,
- Object expected) throws HiveException {
- List<String> mapColumnNames = new ArrayList<String>();
- mapColumnNames.add("A");
- VectorizationContext ctx = new VectorizationContext(mapColumnNames);
-
- GroupByDesc desc = buildGroupByDescType(ctx, aggregateName, "A",
- TypeInfoFactory.getDecimalTypeInfo(30, 4));
+String aggregateName, Iterable<VectorizedRowBatch> data,
+ Object expected) throws HiveException {
+ List<String> mapColumnNames = new ArrayList<String>();
+ mapColumnNames.add("A");
+ VectorizationContext ctx = new VectorizationContext(mapColumnNames);
- VectorGroupByOperator vgo = new VectorGroupByOperator(ctx, desc);
+ GroupByDesc desc =
+ buildGroupByDescType(ctx, aggregateName, "A", TypeInfoFactory.getDecimalTypeInfo(30, 4));
- FakeCaptureOutputOperator out = FakeCaptureOutputOperator.addCaptureOutputChild(vgo);
- vgo.initialize(null, null);
+ VectorGroupByOperator vgo = new VectorGroupByOperator(ctx, desc);
- for (VectorizedRowBatch unit: data) {
- vgo.processOp(unit, 0);
- }
- vgo.close(false);
+ FakeCaptureOutputOperator out = FakeCaptureOutputOperator.addCaptureOutputChild(vgo);
+ vgo.initialize(hconf, null);
- List<Object> outBatchList = out.getCapturedRows();
- assertNotNull(outBatchList);
- assertEquals(1, outBatchList.size());
+ for (VectorizedRowBatch unit : data) {
+ vgo.process(unit, 0);
+ }
+ vgo.close(false);
- Object result = outBatchList.get(0);
+ List<Object> outBatchList = out.getCapturedRows();
+ assertNotNull(outBatchList);
+ assertEquals(1, outBatchList.size());
- Validator validator = getValidator(aggregateName);
- validator.validate("_total", expected, result);
- }
+ Object result = outBatchList.get(0);
+
+ Validator validator = getValidator(aggregateName);
+ validator.validate("_total", expected, result);
+ }
public void testAggregateDoubleIterable (
@@ -2366,10 +2366,10 @@ public class TestVectorGroupByOperator {
VectorGroupByOperator vgo = new VectorGroupByOperator(ctx, desc);
FakeCaptureOutputOperator out = FakeCaptureOutputOperator.addCaptureOutputChild(vgo);
- vgo.initialize(null, null);
+ vgo.initialize(hconf, null);
for (VectorizedRowBatch unit: data) {
- vgo.processOp(unit, 0);
+ vgo.process(unit, 0);
}
vgo.close(false);
@@ -2399,7 +2399,7 @@ public class TestVectorGroupByOperator {
vgo.initialize(null, null);
for (VectorizedRowBatch unit: data) {
- vgo.processOp(unit, 0);
+ vgo.process(unit, 0);
}
vgo.close(false);
@@ -2430,7 +2430,7 @@ public class TestVectorGroupByOperator {
VectorGroupByOperator vgo = new VectorGroupByOperator(ctx, desc);
FakeCaptureOutputOperator out = FakeCaptureOutputOperator.addCaptureOutputChild(vgo);
- vgo.initialize(null, null);
+ vgo.initialize(hconf, null);
out.setOutputInspector(new FakeCaptureOutputOperator.OutputInspector() {
private int rowIndex;
@@ -2469,7 +2469,7 @@ public class TestVectorGroupByOperator {
}.init(aggregateName, expected, keys));
for (VectorizedRowBatch unit: data) {
- vgo.processOp(unit, 0);
+ vgo.process(unit, 0);
}
vgo.close(false);
@@ -2496,7 +2496,7 @@ public class TestVectorGroupByOperator {
VectorGroupByOperator vgo = new VectorGroupByOperator(ctx, desc);
FakeCaptureOutputOperator out = FakeCaptureOutputOperator.addCaptureOutputChild(vgo);
- vgo.initialize(null, null);
+ vgo.initialize(hconf, null);
out.setOutputInspector(new FakeCaptureOutputOperator.OutputInspector() {
private int rowIndex;
@@ -2536,7 +2536,7 @@ public class TestVectorGroupByOperator {
}.init(aggregateName, expected, keys));
for (VectorizedRowBatch unit: data) {
- vgo.processOp(unit, 0);
+ vgo.process(unit, 0);
}
vgo.close(false);
Modified: hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorLimitOperator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorLimitOperator.java?rev=1667454&r1=1667453&r2=1667454&view=diff
==============================================================================
--- hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorLimitOperator.java (original)
+++ hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorLimitOperator.java Wed Mar 18 05:03:23 2015
@@ -67,7 +67,7 @@ public class TestVectorLimitOperator {
lo.initialize(new Configuration(), null);
// Process the batch
- lo.processOp(vrb, 0);
+ lo.process(vrb, 0);
// Verify batch size
Assert.assertEquals(vrb.size, expectedBatchSize);
Modified: hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorSelectOperator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorSelectOperator.java?rev=1667454&r1=1667453&r2=1667454&view=diff
==============================================================================
--- hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorSelectOperator.java (original)
+++ hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorSelectOperator.java Wed Mar 18 05:03:23 2015
@@ -119,7 +119,7 @@ public class TestVectorSelectOperator {
VectorizedRowBatch vrg = VectorizedRowGroupGenUtil.getVectorizedRowBatch(
VectorizedRowBatch.DEFAULT_SIZE, 4, 17);
- vso.processOp(vrg, 0);
+ vso.process(vrg, 0);
}
}
Modified: hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/util/FakeCaptureOutputOperator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/util/FakeCaptureOutputOperator.java?rev=1667454&r1=1667453&r2=1667454&view=diff
==============================================================================
--- hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/util/FakeCaptureOutputOperator.java (original)
+++ hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/util/FakeCaptureOutputOperator.java Wed Mar 18 05:03:23 2015
@@ -20,8 +20,9 @@ package org.apache.hadoop.hive.ql.exec.v
import java.io.Serializable;
import java.util.ArrayList;
-import java.util.Arrays;
+import java.util.Collection;
import java.util.List;
+import java.util.concurrent.Future;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.ql.exec.Operator;
@@ -36,17 +37,17 @@ import org.apache.hadoop.hive.ql.plan.ap
public class FakeCaptureOutputOperator extends Operator<FakeCaptureOutputDesc>
implements Serializable {
private static final long serialVersionUID = 1L;
-
+
public interface OutputInspector {
public void inspectRow(Object row, int tag) throws HiveException;
}
-
+
private OutputInspector outputInspector;
-
+
public void setOutputInspector(OutputInspector outputInspector) {
this.outputInspector = outputInspector;
}
-
+
public OutputInspector getOutputInspector() {
return outputInspector;
}
@@ -67,18 +68,20 @@ public class FakeCaptureOutputOperator e
return out;
}
-
+
public List<Object> getCapturedRows() {
return rows;
}
@Override
- public void initializeOp(Configuration conf) throws HiveException {
+ public Collection<Future<?>> initializeOp(Configuration conf) throws HiveException {
+ Collection<Future<?>> result = super.initializeOp(conf);
rows = new ArrayList<Object>();
+ return result;
}
@Override
- public void processOp(Object row, int tag) throws HiveException {
+ public void process(Object row, int tag) throws HiveException {
rows.add(row);
if (null != outputInspector) {
outputInspector.inspectRow(row, tag);
Modified: hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/util/FakeVectorDataSourceOperator.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/util/FakeVectorDataSourceOperator.java?rev=1667454&r1=1667453&r2=1667454&view=diff
==============================================================================
--- hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/util/FakeVectorDataSourceOperator.java (original)
+++ hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/util/FakeVectorDataSourceOperator.java Wed Mar 18 05:03:23 2015
@@ -20,7 +20,9 @@ package org.apache.hadoop.hive.ql.exec.v
import java.io.Serializable;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.List;
+import java.util.concurrent.Future;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.ql.exec.Operator;
@@ -60,11 +62,12 @@ public class FakeVectorDataSourceOperato
}
@Override
- public void initializeOp(Configuration conf) throws HiveException {
+ public Collection<Future<?>> initializeOp(Configuration conf) throws HiveException {
+ return super.initializeOp(conf);
}
@Override
- public void processOp(Object row, int tag) throws HiveException {
+ public void process(Object row, int tag) throws HiveException {
for (VectorizedRowBatch unit: source) {
forward(unit, null);
}
Modified: hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/testutil/OperatorTestUtils.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/testutil/OperatorTestUtils.java?rev=1667454&r1=1667453&r2=1667454&view=diff
==============================================================================
--- hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/testutil/OperatorTestUtils.java (original)
+++ hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/testutil/OperatorTestUtils.java Wed Mar 18 05:03:23 2015
@@ -67,7 +67,7 @@ public class OperatorTestUtils {
InspectableObject [] sourceData, InspectableObject [] expected) throws HiveException {
InspectableObject resultRef = new InspectableObject();
for (int i = 0; i < sourceData.length; i++) {
- selectOp.processOp(sourceData[i].o, 0);
+ selectOp.process(sourceData[i].o, 0);
collectOp.retrieve(resultRef);
StructObjectInspector expectedOi = (StructObjectInspector) expected[i].oi;
List<? extends StructField> expectedFields = expectedOi.getAllStructFieldRefs();