You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by mm...@apache.org on 2018/03/27 07:28:23 UTC
hive git commit: HIVE-18995: Vectorization: Add option to suppress
"Execution mode: vectorized" for testing purposes (Matt McCline,
reviewed by Teddy Choi)
Repository: hive
Updated Branches:
refs/heads/master fb675ce74 -> 1e884cc5f
HIVE-18995: Vectorization: Add option to suppress "Execution mode: vectorized" for testing purposes (Matt McCline, reviewed by Teddy Choi)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/1e884cc5
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/1e884cc5
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/1e884cc5
Branch: refs/heads/master
Commit: 1e884cc5f5e8d2fa966a604066ac059022ca5649
Parents: fb675ce
Author: Matt McCline <mm...@hortonworks.com>
Authored: Tue Mar 27 02:28:01 2018 -0500
Committer: Matt McCline <mm...@hortonworks.com>
Committed: Tue Mar 27 02:28:01 2018 -0500
----------------------------------------------------------------------
.../org/apache/hadoop/hive/conf/HiveConf.java | 5 +
.../optimizer/physical/PhysicalOptimizer.java | 5 +-
.../hive/ql/optimizer/physical/Vectorizer.java | 234 ++++++++++++-------
.../apache/hadoop/hive/ql/plan/BaseWork.java | 141 +++++++----
.../org/apache/hadoop/hive/ql/plan/MapWork.java | 4 +-
.../hadoop/hive/ql/plan/MergeJoinWork.java | 2 +-
.../apache/hadoop/hive/ql/plan/ReduceWork.java | 16 +-
7 files changed, 268 insertions(+), 139 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/1e884cc5/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 8d9b5a3..5888eaa 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -2999,6 +2999,11 @@ public class HiveConf extends Configuration {
"internal use only, used to override the hive.vectorized.execution.enabled setting and\n" +
"turn off vectorization. The default is false, of course",
true),
+ HIVE_TEST_VECTORIZATION_SUPPRESS_EXPLAIN_EXECUTION_MODE(
+ "hive.test.vectorization.suppress.explain.execution.mode", false,
+ "internal use only, used to suppress \"Execution mode: vectorized\" EXPLAIN display.\n" +
+ "The default is false, of course",
+ true),
HIVE_TYPE_CHECK_ON_INSERT("hive.typecheck.on.insert", true, "This property has been extended to control "
+ "whether to check, convert, and normalize partition value to conform to its column type in "
http://git-wip-us.apache.org/repos/asf/hive/blob/1e884cc5/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/PhysicalOptimizer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/PhysicalOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/PhysicalOptimizer.java
index 0f3c5f2..d508d02 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/PhysicalOptimizer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/PhysicalOptimizer.java
@@ -85,7 +85,10 @@ public class PhysicalOptimizer {
// Vectorization should be the last optimization, because it doesn't modify the plan
// or any operators. It makes a very low level transformation to the expressions to
// run in the vectorized mode.
- if (hiveConf.getBoolVar(HiveConf.ConfVars.HIVE_VECTORIZATION_ENABLED)) {
+ if (hiveConf.getBoolVar(HiveConf.ConfVars.HIVE_VECTORIZATION_ENABLED) ||
+ HiveConf.getVar(hiveConf,
+ HiveConf.ConfVars.HIVE_TEST_VECTORIZATION_ENABLED_OVERRIDE).equalsIgnoreCase(
+ "enable")) {
resolvers.add(new Vectorizer());
}
if (!"none".equalsIgnoreCase(hiveConf.getVar(HiveConf.ConfVars.HIVESTAGEIDREARRANGE))) {
http://git-wip-us.apache.org/repos/asf/hive/blob/1e884cc5/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
index 13a2fc4..0c94688 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
@@ -231,6 +231,7 @@ import org.apache.hadoop.mapred.SequenceFileInputFormat;
import org.apache.hadoop.mapred.TextInputFormat;
import org.apache.hive.common.util.AnnotationUtils;
import org.apache.hadoop.util.ReflectionUtils;
+
import com.google.common.collect.ImmutableSet;
import com.google.common.base.Preconditions;
@@ -300,6 +301,7 @@ public class Vectorizer implements PhysicalPlanResolver {
boolean isVectorizationEnabled;
private VectorizationEnabledOverride vectorizationEnabledOverride;
+ boolean isTestForcedVectorizationEnable;
private boolean useVectorizedInputFileFormat;
private boolean useVectorDeserialize;
@@ -336,6 +338,8 @@ public class Vectorizer implements PhysicalPlanResolver {
supportedAcidInputFormats.add(OneNullRowInputFormat.class.getName());
}
+ private boolean isTestVectorizationSuppressExplainExecutionMode;
+
private BaseWork currentBaseWork;
private Operator<? extends OperatorDesc> currentOperator;
private Collection<Class<?>> vectorizedInputFormatExcludes;
@@ -651,6 +655,10 @@ public class Vectorizer implements PhysicalPlanResolver {
baseWork.setAllNative(allNative);
baseWork.setUsesVectorUDFAdaptor(usesVectorUDFAdaptor);
+
+ baseWork.setIsTestForcedVectorizationEnable(isTestForcedVectorizationEnable);
+ baseWork.setIsTestVectorizationSuppressExplainExecutionMode(
+ isTestVectorizationSuppressExplainExecutionMode);
}
}
@@ -923,19 +931,30 @@ public class Vectorizer implements PhysicalPlanResolver {
Task<? extends Serializable> currTask = (Task<? extends Serializable>) nd;
if (currTask instanceof MapRedTask) {
MapredWork mapredWork = ((MapRedTask) currTask).getWork();
- convertMapWork(mapredWork.getMapWork(), false);
+
+ MapWork mapWork = mapredWork.getMapWork();
+ setMapWorkExplainConditions(mapWork);
+ convertMapWork(mapredWork.getMapWork(), /* isTezOrSpark */ false);
+ logMapWorkExplainVectorization(mapWork);
+
ReduceWork reduceWork = mapredWork.getReduceWork();
if (reduceWork != null) {
+
// Always set the EXPLAIN conditions.
setReduceWorkExplainConditions(reduceWork);
// We do not vectorize MR Reduce.
+
+ logReduceWorkExplainVectorization(reduceWork);
}
} else if (currTask instanceof TezTask) {
TezWork work = ((TezTask) currTask).getWork();
for (BaseWork baseWork: work.getAllWork()) {
if (baseWork instanceof MapWork) {
- convertMapWork((MapWork) baseWork, true);
+ MapWork mapWork = (MapWork) baseWork;
+ setMapWorkExplainConditions(mapWork);
+ convertMapWork(mapWork, /* isTezOrSpark */ true);
+ logMapWorkExplainVectorization(mapWork);
} else if (baseWork instanceof ReduceWork) {
ReduceWork reduceWork = (ReduceWork) baseWork;
@@ -946,13 +965,17 @@ public class Vectorizer implements PhysicalPlanResolver {
if (isReduceVectorizationEnabled) {
convertReduceWork(reduceWork);
}
+ logReduceWorkExplainVectorization(reduceWork);
}
}
} else if (currTask instanceof SparkTask) {
SparkWork sparkWork = (SparkWork) currTask.getWork();
for (BaseWork baseWork : sparkWork.getAllWork()) {
if (baseWork instanceof MapWork) {
- convertMapWork((MapWork) baseWork, true);
+ MapWork mapWork = (MapWork) baseWork;
+ setMapWorkExplainConditions(mapWork);
+ convertMapWork(mapWork, /* isTezOrSpark */ true);
+ logMapWorkExplainVectorization(mapWork);
} else if (baseWork instanceof ReduceWork) {
ReduceWork reduceWork = (ReduceWork) baseWork;
@@ -962,37 +985,125 @@ public class Vectorizer implements PhysicalPlanResolver {
if (isReduceVectorizationEnabled) {
convertReduceWork(reduceWork);
}
+ logReduceWorkExplainVectorization(reduceWork);
}
}
+ } else if (currTask instanceof FetchTask) {
+ LOG.info("Vectorizing Fetch not supported");
+ } else {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Ignoring vectorization of " + currTask.getClass().getSimpleName());
+ }
}
return null;
}
- private void convertMapWork(MapWork mapWork, boolean isTezOrSpark) throws SemanticException {
-
- mapWork.setVectorizationExamined(true);
+ private void setExplainConditions(BaseWork baseWork) {
// Global used when setting errors, etc.
- currentBaseWork = mapWork;
+ currentBaseWork = baseWork;
- VectorTaskColumnInfo vectorTaskColumnInfo = new VectorTaskColumnInfo();
- vectorTaskColumnInfo.assume();
+ baseWork.setVectorizedVertexNum(++vectorizedVertexNum);
+ baseWork.setVectorizationExamined(true);
+ }
- mapWork.setVectorizedVertexNum(++vectorizedVertexNum);
+ private void setMapWorkExplainConditions(MapWork mapWork) {
+ setExplainConditions(mapWork);
+ }
- if (!validateAndVectorizeMapWork(mapWork, vectorTaskColumnInfo, isTezOrSpark)) {
- if (currentBaseWork.getVectorizationEnabled()) {
- VectorizerReason notVectorizedReason = currentBaseWork.getNotVectorizedReason();
- if (notVectorizedReason == null) {
- LOG.info("Cannot vectorize: unknown");
- } else {
- LOG.info("Cannot vectorize: " + notVectorizedReason.toString());
- }
+ private void setReduceWorkExplainConditions(ReduceWork reduceWork) {
+
+ setExplainConditions(reduceWork);
+
+ reduceWork.setReduceVectorizationEnabled(isReduceVectorizationEnabled);
+ reduceWork.setVectorReduceEngine(
+ HiveConf.getVar(hiveConf, HiveConf.ConfVars.HIVE_EXECUTION_ENGINE));
+ }
+
+ private boolean logExplainVectorization(BaseWork baseWork, String name) {
+
+ if (!baseWork.getVectorizationExamined()) {
+ return false;
+ }
+
+ LOG.info(name + " vectorization enabled: " + baseWork.getVectorizationEnabled());
+ boolean isVectorized = baseWork.getVectorMode();
+ LOG.info(name + " vectorized: " + isVectorized);
+ if (!isVectorized) {
+ VectorizerReason notVectorizedReason = baseWork.getNotVectorizedReason();
+ if (notVectorizedReason != null) {
+ LOG.info(name + " notVectorizedReason: " + notVectorizedReason.toString());
+ }
+ }
+ LOG.info(name + " vectorizedVertexNum: " + baseWork.getVectorizedVertexNum());
+
+ if (LOG.isDebugEnabled() && isVectorized) {
+ VectorizedRowBatchCtx batchContext = baseWork.getVectorizedRowBatchCtx();
+ LOG.debug(name + " dataColumnCount: " + batchContext.getDataColumnCount());
+ int[] dataColumnNums = batchContext.getDataColumnNums();
+ if (dataColumnNums != null) {
+ LOG.debug(name + " includeColumns: " + Arrays.toString(dataColumnNums));
}
+ LOG.debug(name + " partitionColumnCount: " + batchContext.getPartitionColumnCount());
+ LOG.debug(name + " dataColumns: " +
+ BaseWork.BaseExplainVectorization.getColumns(
+ batchContext, 0, batchContext.getDataColumnCount()));
+ LOG.debug(name + " scratchColumnTypeNames: " +
+ BaseWork.BaseExplainVectorization.getScratchColumns(batchContext));
+ VirtualColumn[] neededVirtualColumns = batchContext.getNeededVirtualColumns();
+ if (neededVirtualColumns != null && neededVirtualColumns.length != 0) {
+ LOG.debug(name + " neededVirtualColumns: " + Arrays.toString(neededVirtualColumns));
+ }
+ }
+ return true;
+ }
+
+ private void logMapWorkExplainVectorization(MapWork mapWork) {
+
+ if (!logExplainVectorization(mapWork, "Map")) {
+ return;
+ }
+
+ // Conditions.
+ List<String> enabledConditionsMet = mapWork.getVectorizationEnabledConditionsMet();
+ if (enabledConditionsMet != null && !enabledConditionsMet.isEmpty()) {
+ LOG.info("Map enabledConditionsMet: " + enabledConditionsMet.toString());
+ }
+ List<String> enabledConditionsNotMet = mapWork.getVectorizationEnabledConditionsNotMet();
+ if (enabledConditionsNotMet != null && !enabledConditionsNotMet.isEmpty()) {
+ LOG.info("Map enabledConditionsNotMet: " + enabledConditionsNotMet.toString());
+ }
+ Set<String> inputFileFormatClassNameSet =
+ mapWork.getVectorizationInputFileFormatClassNameSet();
+ if (inputFileFormatClassNameSet != null && !inputFileFormatClassNameSet.isEmpty()) {
+ LOG.info("Map inputFileFormatClassNameSet: " + inputFileFormatClassNameSet.toString());
}
}
+ private void logReduceWorkExplainVectorization(ReduceWork reduceWork) {
+
+ if (!logExplainVectorization(reduceWork, "Reduce")) {
+ return;
+ }
+
+ // Conditions.
+ LOG.info("Reducer " + HiveConf.ConfVars.HIVE_VECTORIZATION_REDUCE_ENABLED.varname +
+ ": " + reduceWork.getReduceVectorizationEnabled());
+ LOG.info("Reducer engine: " + reduceWork.getVectorReduceEngine());
+ }
+
+ private void convertMapWork(MapWork mapWork, boolean isTezOrSpark) throws SemanticException {
+
+ // We have to evaluate the input format to see if vectorization is enabled, so
+ // we do not set it right here.
+
+ VectorTaskColumnInfo vectorTaskColumnInfo = new VectorTaskColumnInfo();
+ vectorTaskColumnInfo.assume();
+
+ validateAndVectorizeMapWork(mapWork, vectorTaskColumnInfo, isTezOrSpark);
+ }
+
private void addMapWorkRules(Map<Rule, NodeProcessor> opRules, NodeProcessor np) {
opRules.put(new RuleRegExp("R1", TableScanOperator.getOperatorName() + ".*"
+ FileSinkOperator.getOperatorName()), np);
@@ -1595,7 +1706,7 @@ public class Vectorizer implements PhysicalPlanResolver {
return new ImmutablePair<Boolean,Boolean>(true, false);
}
- private boolean validateAndVectorizeMapWork(MapWork mapWork, VectorTaskColumnInfo vectorTaskColumnInfo,
+ private void validateAndVectorizeMapWork(MapWork mapWork, VectorTaskColumnInfo vectorTaskColumnInfo,
boolean isTezOrSpark) throws SemanticException {
//--------------------------------------------------------------------------------------------
@@ -1607,7 +1718,7 @@ public class Vectorizer implements PhysicalPlanResolver {
VectorizerReason notVectorizedReason = currentBaseWork.getNotVectorizedReason();
Preconditions.checkState(notVectorizedReason != null);
mapWork.setVectorizationEnabledConditionsNotMet(Arrays.asList(new String[] {notVectorizedReason.toString()}));
- return false;
+ return;
}
String alias = onlyOneTableScanPair.left;
TableScanOperator tableScanOperator = onlyOneTableScanPair.right;
@@ -1625,7 +1736,7 @@ public class Vectorizer implements PhysicalPlanResolver {
Preconditions.checkState(notVectorizedReason != null);
mapWork.setVectorizationEnabledConditionsNotMet(Arrays.asList(new String[] {notVectorizedReason.toString()}));
}
- return false;
+ return;
}
final int dataColumnCount =
@@ -1705,7 +1816,7 @@ public class Vectorizer implements PhysicalPlanResolver {
* Validate and vectorize the Map operator tree.
*/
if (!validateAndVectorizeMapOperators(mapWork, tableScanOperator, isTezOrSpark, vectorTaskColumnInfo)) {
- return false;
+ return;
}
//--------------------------------------------------------------------------------------------
@@ -1714,11 +1825,7 @@ public class Vectorizer implements PhysicalPlanResolver {
mapWork.setVectorMode(true);
- if (LOG.isDebugEnabled()) {
- debugDisplayVertexInfo(mapWork);
- }
-
- return true;
+ return;
}
private boolean validateAndVectorizeMapOperators(MapWork mapWork, TableScanOperator tableScanOperator,
@@ -1859,47 +1966,26 @@ public class Vectorizer implements PhysicalPlanResolver {
return newChildren;
}
- private void setReduceWorkExplainConditions(ReduceWork reduceWork) {
-
- reduceWork.setVectorizationExamined(true);
-
- reduceWork.setReduceVectorizationEnabled(isReduceVectorizationEnabled);
- reduceWork.setVectorReduceEngine(
- HiveConf.getVar(hiveConf, HiveConf.ConfVars.HIVE_EXECUTION_ENGINE));
- }
-
private void convertReduceWork(ReduceWork reduceWork) throws SemanticException {
- // Global used when setting errors, etc.
- currentBaseWork = reduceWork;
- currentBaseWork.setVectorizationEnabled(true);
+ reduceWork.setVectorizationEnabled(true);
VectorTaskColumnInfo vectorTaskColumnInfo = new VectorTaskColumnInfo();
vectorTaskColumnInfo.assume();
- reduceWork.setVectorizedVertexNum(++vectorizedVertexNum);
reduceWork.setVectorizedTestingReducerBatchSize(vectorizedTestingReducerBatchSize);
- if (!validateAndVectorizeReduceWork(reduceWork, vectorTaskColumnInfo)) {
- if (currentBaseWork.getVectorizationEnabled()) {
- VectorizerReason notVectorizedReason = currentBaseWork.getNotVectorizedReason();
- if (notVectorizedReason == null) {
- LOG.info("Cannot vectorize: unknown");
- } else {
- LOG.info("Cannot vectorize: " + notVectorizedReason.toString());
- }
- }
- }
+ validateAndVectorizeReduceWork(reduceWork, vectorTaskColumnInfo);
}
- private boolean validateAndVectorizeReduceWork(ReduceWork reduceWork,
+ private void validateAndVectorizeReduceWork(ReduceWork reduceWork,
VectorTaskColumnInfo vectorTaskColumnInfo) throws SemanticException {
Operator<? extends OperatorDesc> reducer = reduceWork.getReducer();
// Validate input to ReduceWork.
if (!getOnlyStructObjectInspectors(reduceWork, vectorTaskColumnInfo)) {
- return false;
+ return;
}
//--------------------------------------------------------------------------------------------
@@ -1908,7 +1994,7 @@ public class Vectorizer implements PhysicalPlanResolver {
* Validate and vectorize the Reduce operator tree.
*/
if (!validateAndVectorizeReduceOperators(reduceWork, vectorTaskColumnInfo)) {
- return false;
+ return;
}
//--------------------------------------------------------------------------------------------
@@ -1917,11 +2003,7 @@ public class Vectorizer implements PhysicalPlanResolver {
reduceWork.setVectorMode(true);
- if (LOG.isDebugEnabled()) {
- debugDisplayVertexInfo(reduceWork);
- }
-
- return true;
+ return;
}
private boolean validateAndVectorizeReduceOperators(ReduceWork reduceWork,
@@ -2073,6 +2155,7 @@ public class Vectorizer implements PhysicalPlanResolver {
HiveConf.ConfVars.HIVE_VECTORIZATION_ENABLED);
final boolean weCanAttemptVectorization;
+ isTestForcedVectorizationEnable = false;
switch (vectorizationEnabledOverride) {
case NONE:
weCanAttemptVectorization = isVectorizationEnabled;
@@ -2082,6 +2165,12 @@ public class Vectorizer implements PhysicalPlanResolver {
break;
case ENABLE:
weCanAttemptVectorization = true;
+ isTestForcedVectorizationEnable = !isVectorizationEnabled;
+
+ // Different parts of the code rely on this being set...
+ HiveConf.setBoolVar(hiveConf,
+ HiveConf.ConfVars.HIVE_VECTORIZATION_ENABLED, true);
+ isVectorizationEnabled = true;
break;
default:
throw new RuntimeException("Unexpected vectorization enabled override " +
@@ -2162,6 +2251,10 @@ public class Vectorizer implements PhysicalPlanResolver {
hiveVectorAdaptorUsageMode = HiveVectorAdaptorUsageMode.getHiveConfValue(hiveConf);
+ isTestVectorizationSuppressExplainExecutionMode =
+ HiveConf.getBoolVar(hiveConf,
+ HiveConf.ConfVars.HIVE_TEST_VECTORIZATION_SUPPRESS_EXPLAIN_EXECUTION_MODE);
+
// create dispatcher and graph walker
Dispatcher disp = new VectorizationDispatcher();
TaskGraphWalker ogw = new TaskGraphWalker(disp);
@@ -2240,7 +2333,7 @@ public class Vectorizer implements PhysicalPlanResolver {
return false;
}
if (desc.getResidualFilterExprs() != null && !desc.getResidualFilterExprs().isEmpty()) {
- LOG.info("Cannot vectorize join with complex ON clause");
+ setOperatorIssue("Non-equi joins not supported");
return false;
}
return true;
@@ -4778,27 +4871,4 @@ public class Vectorizer implements PhysicalPlanResolver {
return vectorOp;
}
-
- public void debugDisplayVertexInfo(BaseWork work) {
-
- VectorizedRowBatchCtx vectorizedRowBatchCtx = work.getVectorizedRowBatchCtx();
-
- String[] allColumnNames = vectorizedRowBatchCtx.getRowColumnNames();
- TypeInfo[] columnTypeInfos = vectorizedRowBatchCtx.getRowColumnTypeInfos();
- DataTypePhysicalVariation[] dataTypePhysicalVariations = vectorizedRowBatchCtx.getRowdataTypePhysicalVariations();
- int partitionColumnCount = vectorizedRowBatchCtx.getPartitionColumnCount();
- int virtualColumnCount = vectorizedRowBatchCtx.getVirtualColumnCount();
- String[] scratchColumnTypeNames =vectorizedRowBatchCtx.getScratchColumnTypeNames();
- DataTypePhysicalVariation[] scratchdataTypePhysicalVariations = vectorizedRowBatchCtx.getScratchDataTypePhysicalVariations();
-
- LOG.debug("debugDisplayVertexInfo rowColumnNames " + Arrays.toString(allColumnNames));
- LOG.debug("debugDisplayVertexInfo rowColumnTypeInfos " + Arrays.toString(columnTypeInfos));
- LOG.debug("debugDisplayVertexInfo rowDataTypePhysicalVariations " +
- (dataTypePhysicalVariations == null ? "NULL" : Arrays.toString(dataTypePhysicalVariations)));
- LOG.debug("debugDisplayVertexInfo partitionColumnCount " + partitionColumnCount);
- LOG.debug("debugDisplayVertexInfo virtualColumnCount " + virtualColumnCount);
- LOG.debug("debugDisplayVertexInfo scratchColumnTypeNames " + Arrays.toString(scratchColumnTypeNames));
- LOG.debug("debugDisplayVertexInfo scratchdataTypePhysicalVariations " +
- (scratchdataTypePhysicalVariations == null ? "NULL" : Arrays.toString(scratchdataTypePhysicalVariations)));
- }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/1e884cc5/ql/src/java/org/apache/hadoop/hive/ql/plan/BaseWork.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/BaseWork.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/BaseWork.java
index dc3219b..31f54c3 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/BaseWork.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/BaseWork.java
@@ -99,6 +99,9 @@ public abstract class BaseWork extends AbstractOperatorDesc {
protected long vectorizedVertexNum;
protected int vectorizedTestingReducerBatchSize;
+ private boolean isTestForcedVectorizationEnable;
+ private boolean isTestVectorizationSuppressExplainExecutionMode;
+
protected boolean llapMode = false;
protected boolean uberMode = false;
@@ -262,6 +265,24 @@ public abstract class BaseWork extends AbstractOperatorDesc {
return allNative;
}
+ public void setIsTestForcedVectorizationEnable(boolean isTestForcedVectorizationEnable) {
+ this.isTestForcedVectorizationEnable = isTestForcedVectorizationEnable;
+ }
+
+ public boolean getIsTestForcedVectorizationEnable() {
+ return isTestForcedVectorizationEnable;
+ }
+
+ public void setIsTestVectorizationSuppressExplainExecutionMode(
+ boolean isTestVectorizationSuppressExplainExecutionMode) {
+ this.isTestVectorizationSuppressExplainExecutionMode =
+ isTestVectorizationSuppressExplainExecutionMode;
+ }
+
+ public boolean getIsTestVectorizationSuppressExplainExecutionMode() {
+ return isTestVectorizationSuppressExplainExecutionMode;
+ }
+
public static class BaseExplainVectorization {
private final BaseWork baseWork;
@@ -287,12 +308,14 @@ public abstract class BaseWork extends AbstractOperatorDesc {
return result;
}
- @Explain(vectorization = Vectorization.SUMMARY, displayName = "enabled", explainLevels = { Level.DEFAULT, Level.EXTENDED })
+ @Explain(vectorization = Vectorization.SUMMARY, displayName = "enabled",
+ explainLevels = { Level.DEFAULT, Level.EXTENDED })
public boolean enabled() {
return baseWork.getVectorizationEnabled();
}
- @Explain(vectorization = Vectorization.SUMMARY, displayName = "vectorized", explainLevels = { Level.DEFAULT, Level.EXTENDED })
+ @Explain(vectorization = Vectorization.SUMMARY, displayName = "vectorized",
+ explainLevels = { Level.DEFAULT, Level.EXTENDED })
public Boolean vectorized() {
if (!baseWork.getVectorizationEnabled()) {
return null;
@@ -300,7 +323,8 @@ public abstract class BaseWork extends AbstractOperatorDesc {
return baseWork.getVectorMode();
}
- @Explain(vectorization = Vectorization.SUMMARY, displayName = "notVectorizedReason", explainLevels = { Level.DEFAULT, Level.EXTENDED })
+ @Explain(vectorization = Vectorization.SUMMARY, displayName = "notVectorizedReason",
+ explainLevels = { Level.DEFAULT, Level.EXTENDED })
public String notVectorizedReason() {
if (!baseWork.getVectorizationEnabled() || baseWork.getVectorMode()) {
return null;
@@ -312,7 +336,8 @@ public abstract class BaseWork extends AbstractOperatorDesc {
return notVectorizedReason.toString();
}
- @Explain(vectorization = Vectorization.SUMMARY, displayName = "allNative", explainLevels = { Level.DEFAULT, Level.EXTENDED })
+ @Explain(vectorization = Vectorization.SUMMARY, displayName = "allNative",
+ explainLevels = { Level.DEFAULT, Level.EXTENDED })
public Boolean nativeVectorized() {
if (!baseWork.getVectorMode()) {
return null;
@@ -320,7 +345,45 @@ public abstract class BaseWork extends AbstractOperatorDesc {
return baseWork.getAllNative();
}
- @Explain(vectorization = Vectorization.SUMMARY, displayName = "usesVectorUDFAdaptor", explainLevels = { Level.DEFAULT, Level.EXTENDED })
+ public static List<String> getColumns(VectorizedRowBatchCtx vectorizedRowBatchCtx,
+ int startIndex, int count) {
+ String[] rowColumnNames = vectorizedRowBatchCtx.getRowColumnNames();
+ TypeInfo[] rowColumnTypeInfos = vectorizedRowBatchCtx.getRowColumnTypeInfos();
+ DataTypePhysicalVariation[] dataTypePhysicalVariations =
+ vectorizedRowBatchCtx.getRowdataTypePhysicalVariations();
+
+ List<String> result = new ArrayList<String>(count);
+ final int end = startIndex + count;
+ for (int i = startIndex; i < end; i++) {
+ String displayString = rowColumnNames[i] + ":" + rowColumnTypeInfos[i];
+ if (dataTypePhysicalVariations != null &&
+ dataTypePhysicalVariations[i] != DataTypePhysicalVariation.NONE) {
+ displayString += "/" + dataTypePhysicalVariations[i].toString();
+ }
+ result.add(displayString);
+ }
+ return result;
+ }
+
+ public static String getScratchColumns(VectorizedRowBatchCtx vectorizedRowBatchCtx) {
+ String[] scratchColumnTypeNames = vectorizedRowBatchCtx.getScratchColumnTypeNames();
+ DataTypePhysicalVariation[] scratchDataTypePhysicalVariations =
+ vectorizedRowBatchCtx.getScratchDataTypePhysicalVariations();
+ final int size = scratchColumnTypeNames.length;
+ List<String> result = new ArrayList<String>(size);
+ for (int i = 0; i < size; i++) {
+ String displayString = scratchColumnTypeNames[i];
+ if (scratchDataTypePhysicalVariations != null &&
+ scratchDataTypePhysicalVariations[i] != DataTypePhysicalVariation.NONE) {
+ displayString += "/" + scratchDataTypePhysicalVariations[i].toString();
+ }
+ result.add(displayString);
+ }
+ return result.toString();
+ }
+
+ @Explain(vectorization = Vectorization.SUMMARY, displayName = "usesVectorUDFAdaptor",
+ explainLevels = { Level.DEFAULT, Level.EXTENDED })
public Boolean usesVectorUDFAdaptor() {
if (!baseWork.getVectorMode()) {
return null;
@@ -336,71 +399,54 @@ public abstract class BaseWork extends AbstractOperatorDesc {
this.vectorizedRowBatchCtx = vectorizedRowBatchCtx;
}
- private List<String> getColumns(int startIndex, int count) {
- String[] rowColumnNames = vectorizedRowBatchCtx.getRowColumnNames();
- TypeInfo[] rowColumnTypeInfos = vectorizedRowBatchCtx.getRowColumnTypeInfos();
- DataTypePhysicalVariation[] dataTypePhysicalVariations =
- vectorizedRowBatchCtx.getRowdataTypePhysicalVariations();
-
- List<String> result = new ArrayList<String>(count);
- final int end = startIndex + count;
- for (int i = startIndex; i < end; i++) {
- String displayString = rowColumnNames[i] + ":" + rowColumnTypeInfos[i];
- if (dataTypePhysicalVariations != null &&
- dataTypePhysicalVariations[i] != DataTypePhysicalVariation.NONE) {
- displayString += "/" + dataTypePhysicalVariations[i].toString();
- }
- result.add(displayString);
- }
- return result;
- }
-
- @Explain(vectorization = Vectorization.DETAIL, displayName = "dataColumns", explainLevels = { Level.DEFAULT, Level.EXTENDED })
+ @Explain(vectorization = Vectorization.DETAIL, displayName = "dataColumns",
+ explainLevels = { Level.DEFAULT, Level.EXTENDED })
public List<String> getDataColumns() {
- return getColumns(0, vectorizedRowBatchCtx.getDataColumnCount());
+ return getColumns(
+ vectorizedRowBatchCtx,
+ 0,
+ vectorizedRowBatchCtx.getDataColumnCount());
}
- @Explain(vectorization = Vectorization.DETAIL, displayName = "partitionColumns", explainLevels = { Level.DEFAULT, Level.EXTENDED })
+ @Explain(vectorization = Vectorization.DETAIL, displayName = "partitionColumns",
+ explainLevels = { Level.DEFAULT, Level.EXTENDED })
public List<String> getPartitionColumns() {
- return getColumns(vectorizedRowBatchCtx.getDataColumnCount(), vectorizedRowBatchCtx.getPartitionColumnCount());
+ return getColumns(
+ vectorizedRowBatchCtx,
+ vectorizedRowBatchCtx.getDataColumnCount(),
+ vectorizedRowBatchCtx.getPartitionColumnCount());
}
- @Explain(vectorization = Vectorization.DETAIL, displayName = "includeColumns", explainLevels = { Level.DEFAULT, Level.EXTENDED })
+ @Explain(vectorization = Vectorization.DETAIL, displayName = "includeColumns",
+ explainLevels = { Level.DEFAULT, Level.EXTENDED })
public String getDataColumnNums() {
int[] dataColumnNums = vectorizedRowBatchCtx.getDataColumnNums();
if (dataColumnNums == null) {
return null;
}
- return Arrays.toString(vectorizedRowBatchCtx.getDataColumnNums());
+ return Arrays.toString(dataColumnNums);
}
- @Explain(vectorization = Vectorization.DETAIL, displayName = "dataColumnCount", explainLevels = { Level.DEFAULT, Level.EXTENDED })
+ @Explain(vectorization = Vectorization.DETAIL, displayName = "dataColumnCount",
+ explainLevels = { Level.DEFAULT, Level.EXTENDED })
public int getDataColumnCount() {
return vectorizedRowBatchCtx.getDataColumnCount();
}
- @Explain(vectorization = Vectorization.DETAIL, displayName = "partitionColumnCount", explainLevels = { Level.DEFAULT, Level.EXTENDED })
+ @Explain(vectorization = Vectorization.DETAIL, displayName = "partitionColumnCount",
+ explainLevels = { Level.DEFAULT, Level.EXTENDED })
public int getPartitionColumnCount() {
return vectorizedRowBatchCtx.getPartitionColumnCount();
}
- @Explain(vectorization = Vectorization.DETAIL, displayName = "scratchColumnTypeNames", explainLevels = { Level.DEFAULT, Level.EXTENDED })
+ @Explain(vectorization = Vectorization.DETAIL, displayName = "scratchColumnTypeNames",
+ explainLevels = { Level.DEFAULT, Level.EXTENDED })
public String getScratchColumnTypeNames() {
- String[] scratchColumnTypeNames = vectorizedRowBatchCtx.getScratchColumnTypeNames();
- DataTypePhysicalVariation[] scratchDataTypePhysicalVariations = vectorizedRowBatchCtx.getScratchDataTypePhysicalVariations();
- final int size = scratchColumnTypeNames.length;
- List<String> result = new ArrayList<String>(size);
- for (int i = 0; i < size; i++) {
- String displayString = scratchColumnTypeNames[i];
- if (scratchDataTypePhysicalVariations != null && scratchDataTypePhysicalVariations[i] != DataTypePhysicalVariation.NONE) {
- displayString += "/" + scratchDataTypePhysicalVariations[i].toString();
- }
- result.add(displayString);
- }
- return result.toString();
+ return getScratchColumns(vectorizedRowBatchCtx);
}
- @Explain(vectorization = Vectorization.DETAIL, displayName = "neededVirtualColumns", explainLevels = { Level.DEFAULT, Level.EXTENDED })
+ @Explain(vectorization = Vectorization.DETAIL, displayName = "neededVirtualColumns",
+ explainLevels = { Level.DEFAULT, Level.EXTENDED })
public String getNeededVirtualColumns() {
VirtualColumn[] neededVirtualColumns = vectorizedRowBatchCtx.getNeededVirtualColumns();
if (neededVirtualColumns == null || neededVirtualColumns.length == 0) {
@@ -411,7 +457,8 @@ public abstract class BaseWork extends AbstractOperatorDesc {
}
- @Explain(vectorization = Vectorization.DETAIL, displayName = "rowBatchContext", explainLevels = { Level.DEFAULT, Level.EXTENDED })
+ @Explain(vectorization = Vectorization.DETAIL, displayName = "rowBatchContext",
+ explainLevels = { Level.DEFAULT, Level.EXTENDED })
public RowBatchContextExplainVectorization vectorizedRowBatchContext() {
if (!baseWork.getVectorMode()) {
return null;
http://git-wip-us.apache.org/repos/asf/hive/blob/1e884cc5/ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java
index f147309..5bf0625 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/MapWork.java
@@ -468,7 +468,9 @@ public class MapWork extends BaseWork {
@Explain(displayName = "Execution mode", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED },
vectorization = Vectorization.SUMMARY_PATH)
public String getExecutionMode() {
- if (vectorMode) {
+ if (vectorMode &&
+ !(getIsTestForcedVectorizationEnable() &&
+ getIsTestVectorizationSuppressExplainExecutionMode())) {
if (llapMode) {
if (uberMode) {
return "vectorized, uber";
http://git-wip-us.apache.org/repos/asf/hive/blob/1e884cc5/ql/src/java/org/apache/hadoop/hive/ql/plan/MergeJoinWork.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/MergeJoinWork.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/MergeJoinWork.java
index b0ae64a..24ce898 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/MergeJoinWork.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/MergeJoinWork.java
@@ -175,7 +175,7 @@ public class MergeJoinWork extends BaseWork {
public boolean getLlapMode() {
return getMainWork().getLlapMode();
}
-
+
public void addDummyOp(HashTableDummyOperator dummyOp) {
getMainWork().addDummyOp(dummyOp);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/1e884cc5/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceWork.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceWork.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceWork.java
index ff5acbb..51298ce 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceWork.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceWork.java
@@ -126,15 +126,17 @@ public class ReduceWork extends BaseWork {
@Explain(displayName = "Execution mode", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED },
vectorization = Vectorization.SUMMARY_PATH)
public String getExecutionMode() {
- if (vectorMode) {
+ if (vectorMode &&
+ !(getIsTestForcedVectorizationEnable() &&
+ getIsTestVectorizationSuppressExplainExecutionMode())) {
if (llapMode) {
- if (uberMode) {
- return "vectorized, uber";
- } else {
- return "vectorized, llap";
- }
+ if (uberMode) {
+ return "vectorized, uber";
+ } else {
+ return "vectorized, llap";
+ }
} else {
- return "vectorized";
+ return "vectorized";
}
} else if (llapMode) {
return uberMode? "uber" : "llap";