You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ab...@apache.org on 2019/08/27 12:00:48 UTC
[hive] branch master updated: HIVE-21944: Remove unused methods,
fields and variables from Vectorizer (Ivan Suller via Laszlo Bodor)
This is an automated email from the ASF dual-hosted git repository.
abstractdog pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new d26516e HIVE-21944: Remove unused methods, fields and variables from Vectorizer (Ivan Suller via Laszlo Bodor)
d26516e is described below
commit d26516e7e18d2a729a681ee23f5e13c7282af94e
Author: Ivan Suller <is...@cloudera.com>
AuthorDate: Tue Aug 27 13:53:22 2019 +0200
HIVE-21944: Remove unused methods, fields and variables from Vectorizer (Ivan Suller via Laszlo Bodor)
Signed-off-by: Laszlo Bodor <bo...@gmail.com>
---
.../hive/ql/optimizer/physical/Vectorizer.java | 71 ++--------------------
1 file changed, 6 insertions(+), 65 deletions(-)
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
index ed3db4c..b650299 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
@@ -104,9 +104,6 @@ import org.apache.hadoop.hive.ql.io.ZeroRowsInputFormat;
import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatchCtx;
import org.apache.hadoop.hive.ql.lib.Dispatcher;
import org.apache.hadoop.hive.ql.lib.Node;
-import org.apache.hadoop.hive.ql.lib.NodeProcessor;
-import org.apache.hadoop.hive.ql.lib.Rule;
-import org.apache.hadoop.hive.ql.lib.RuleRegExp;
import org.apache.hadoop.hive.ql.lib.TaskGraphWalker;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.metadata.VirtualColumn;
@@ -312,9 +309,9 @@ public class Vectorizer implements PhysicalPlanResolver {
};
}
- boolean isVectorizationEnabled;
+ private boolean isVectorizationEnabled;
private EnabledOverride vectorizationEnabledOverride;
- boolean isTestForcedVectorizationEnable;
+ private boolean isTestForcedVectorizationEnable;
private boolean useVectorizedInputFileFormat;
private boolean useVectorDeserialize;
@@ -858,10 +855,7 @@ public class Vectorizer implements PhysicalPlanResolver {
VectorTaskColumnInfo vectorTaskColumnInfo)
throws VectorizerCannotVectorizeException {
- List<Operator<? extends OperatorDesc>> vectorChildren = newOperatorList();
List<Operator<? extends OperatorDesc>> children = parent.getChildOperators();
- List<List<Operator<? extends OperatorDesc>>> listOfChildMultipleParents =
- new ArrayList<List<Operator<? extends OperatorDesc>>>();
final int childrenCount = children.size();
for (int i = 0; i < childrenCount; i++) {
@@ -977,7 +971,6 @@ public class Vectorizer implements PhysicalPlanResolver {
vContext = ((VectorizationOperator) vectorParent).getInputVectorizationContext();
}
- OperatorDesc desc = child.getConf();
Operator<? extends OperatorDesc> vectorChild;
try {
@@ -1193,13 +1186,6 @@ public class Vectorizer implements PhysicalPlanResolver {
validateAndVectorizeMapWork(mapWork, vectorTaskColumnInfo, isTezOrSpark);
}
- private void addMapWorkRules(Map<Rule, NodeProcessor> opRules, NodeProcessor np) {
- opRules.put(new RuleRegExp("R1", TableScanOperator.getOperatorName() + ".*"
- + FileSinkOperator.getOperatorName()), np);
- opRules.put(new RuleRegExp("R2", TableScanOperator.getOperatorName() + ".*"
- + ReduceSinkOperator.getOperatorName()), np);
- }
-
/*
* Determine if there is only one TableScanOperator. Currently in Map vectorization, we do not
* try to vectorize multiple input trees.
@@ -1232,7 +1218,7 @@ public class Vectorizer implements PhysicalPlanResolver {
setNodeIssue("Vectorized map work only works with 1 TableScanOperator");
return null;
}
- return new ImmutablePair(alias, tableScanOperator);
+ return new ImmutablePair<>(alias, tableScanOperator);
}
private void getTableScanOperatorSchemaInfo(TableScanOperator tableScanOperator,
@@ -2051,8 +2037,6 @@ public class Vectorizer implements PhysicalPlanResolver {
vectorTaskColumnInfo.transferToBaseWork(mapWork);
mapWork.setVectorMode(true);
-
- return;
}
private boolean validateAndVectorizeMapOperators(MapWork mapWork, TableScanOperator tableScanOperator,
@@ -2227,8 +2211,6 @@ public class Vectorizer implements PhysicalPlanResolver {
private void validateAndVectorizeReduceWork(ReduceWork reduceWork,
VectorTaskColumnInfo vectorTaskColumnInfo) throws SemanticException {
- Operator<? extends OperatorDesc> reducer = reduceWork.getReducer();
-
// Validate input to ReduceWork.
if (!getOnlyStructObjectInspectors(reduceWork, vectorTaskColumnInfo)) {
return;
@@ -2613,18 +2595,6 @@ public class Vectorizer implements PhysicalPlanResolver {
return true;
}
- private boolean validateSparkHashTableSinkOperator(SparkHashTableSinkOperator op) {
- SparkHashTableSinkDesc desc = op.getConf();
- byte tag = desc.getTag();
- // it's essentially a MapJoinDesc
- List<ExprNodeDesc> filterExprs = desc.getFilters().get(tag);
- List<ExprNodeDesc> keyExprs = desc.getKeys().get(tag);
- List<ExprNodeDesc> valueExprs = desc.getExprs().get(tag);
- return validateExprNodeDesc(
- filterExprs, "Filter", VectorExpressionDescriptor.Mode.FILTER, /* allowComplex */ true) &&
- validateExprNodeDesc(keyExprs, "Key") && validateExprNodeDesc(valueExprs, "Value");
- }
-
private boolean validateReduceSinkOperator(ReduceSinkOperator op) {
List<ExprNodeDesc> keyDescs = op.getConf().getKeyCols();
List<ExprNodeDesc> partitionDescs = op.getConf().getPartitionCols();
@@ -2902,7 +2872,6 @@ public class Vectorizer implements PhysicalPlanResolver {
}
}
- boolean isPartitionOrderBy = vectorPTFDesc.getIsPartitionOrderBy();
String[] evaluatorFunctionNames = vectorPTFDesc.getEvaluatorFunctionNames();
final int count = evaluatorFunctionNames.length;
WindowFrameDef[] evaluatorWindowFrameDefs = vectorPTFDesc.getEvaluatorWindowFrameDefs();
@@ -3279,20 +3248,6 @@ public class Vectorizer implements PhysicalPlanResolver {
return (result ? null : "Vectorizing data type " + type + " not supported");
}
- private VectorizationContext getVectorizationContext(String contextName,
- VectorTaskColumnInfo vectorTaskColumnInfo) {
-
- VectorizationContext vContext =
- new VectorizationContext(
- contextName,
- vectorTaskColumnInfo.allColumnNames,
- vectorTaskColumnInfo.allTypeInfos,
- vectorTaskColumnInfo.allDataTypePhysicalVariations,
- hiveConf);
-
- return vContext;
- }
-
private void fixupParentChildOperators(Operator<? extends OperatorDesc> op,
Operator<? extends OperatorDesc> vectorOp) {
if (op.getParentOperators() != null) {
@@ -3783,16 +3738,13 @@ public class Vectorizer implements PhysicalPlanResolver {
int firstSmallTableOutputColumn;
firstSmallTableOutputColumn = (order[0] == posBigTable ? bigTableRetainSize : 0);
- int smallTableOutputCount = 0;
nextOutputColumn = firstSmallTableOutputColumn;
// Small table indices has more information (i.e. keys) than retain, so use it if it exists...
if (smallTableIndicesSize > 0) {
- smallTableOutputCount = smallTableIndicesSize;
for (int i = 0; i < smallTableIndicesSize; i++) {
if (smallTableIndices[i] >= 0) {
-
// Zero and above numbers indicate a big table key is needed for
// small table result "area".
@@ -3856,8 +3808,6 @@ public class Vectorizer implements PhysicalPlanResolver {
nextOutputColumn++;
}
} else if (smallTableRetainSize > 0) {
- smallTableOutputCount = smallTableRetainSize;
-
// Only small table values appear in join output result.
for (int i = 0; i < smallTableRetainSize; i++) {
@@ -3906,11 +3856,11 @@ public class Vectorizer implements PhysicalPlanResolver {
vectorDesc.setSupportsKeyTypes(supportsKeyTypes);
if (!supportsKeyTypes) {
- vectorDesc.setNotSupportedKeyTypes(new ArrayList(notSupportedKeyTypes));
+ vectorDesc.setNotSupportedKeyTypes(new ArrayList<>(notSupportedKeyTypes));
}
vectorDesc.setSupportsValueTypes(supportsValueTypes);
if (!supportsValueTypes) {
- vectorDesc.setNotSupportedValueTypes(new ArrayList(notSupportedValueTypes));
+ vectorDesc.setNotSupportedValueTypes(new ArrayList<>(notSupportedValueTypes));
}
// Check common conditions for both Optimized and Fast Hash Tables.
@@ -4995,9 +4945,7 @@ public class Vectorizer implements PhysicalPlanResolver {
PTFDesc ptfDesc, VectorizationContext vContext, VectorPTFDesc vectorPTFDesc)
throws HiveException {
- PartitionedTableFunctionDef funcDef = ptfDesc.getFuncDef();
-
- ArrayList<ColumnInfo> outputSignature = ptfOp.getSchema().getSignature();
+ List<ColumnInfo> outputSignature = ptfOp.getSchema().getSignature();
final int outputSize = outputSignature.size();
boolean isPartitionOrderBy = vectorPTFDesc.getIsPartitionOrderBy();
@@ -5006,7 +4954,6 @@ public class Vectorizer implements PhysicalPlanResolver {
String[] evaluatorFunctionNames = vectorPTFDesc.getEvaluatorFunctionNames();
final int evaluatorCount = evaluatorFunctionNames.length;
- WindowFrameDef[] evaluatorWindowFrameDefs = vectorPTFDesc.getEvaluatorWindowFrameDefs();
List<ExprNodeDesc>[] evaluatorInputExprNodeDescLists = vectorPTFDesc.getEvaluatorInputExprNodeDescLists();
/*
@@ -5080,10 +5027,6 @@ public class Vectorizer implements PhysicalPlanResolver {
VectorExpression[] evaluatorInputExpressions = new VectorExpression[evaluatorCount];
Type[] evaluatorInputColumnVectorTypes = new Type[evaluatorCount];
for (int i = 0; i < evaluatorCount; i++) {
- String functionName = evaluatorFunctionNames[i];
- WindowFrameDef windowFrameDef = evaluatorWindowFrameDefs[i];
- SupportedFunctionType functionType = VectorPTFDesc.supportedFunctionsMap.get(functionName);
-
List<ExprNodeDesc> exprNodeDescList = evaluatorInputExprNodeDescLists[i];
VectorExpression inputVectorExpression;
final Type columnVectorType;
@@ -5096,7 +5039,6 @@ public class Vectorizer implements PhysicalPlanResolver {
inputVectorExpression = vContext.getVectorExpression(exprNodeDesc);
TypeInfo typeInfo = exprNodeDesc.getTypeInfo();
- PrimitiveCategory primitiveCategory = ((PrimitiveTypeInfo) typeInfo).getPrimitiveCategory();
columnVectorType = VectorizationContext.getColumnVectorTypeFromTypeInfo(typeInfo);
} else {
inputVectorExpression = null;
@@ -5223,7 +5165,6 @@ public class Vectorizer implements PhysicalPlanResolver {
isNative = true;
if (vectorTaskColumnInfo != null) {
- VectorMapJoinInfo vectorMapJoinInfo = vectorMapJoinDesc.getVectorMapJoinInfo();
if (usesVectorUDFAdaptor(vectorMapJoinDesc.getAllBigTableKeyExpressions())) {
vectorTaskColumnInfo.setUsesVectorUDFAdaptor(true);
}