You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by mm...@apache.org on 2018/09/16 14:05:46 UTC
[42/47] hive git commit: HIVE-18908: FULL OUTER JOIN to MapJoin (Matt
McCline, reviewed by Teddy Choi)
http://git-wip-us.apache.org/repos/asf/hive/blob/a37827ec/ql/src/java/org/apache/hadoop/hive/ql/optimizer/FullOuterMapJoinOptimization.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/FullOuterMapJoinOptimization.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/FullOuterMapJoinOptimization.java
new file mode 100644
index 0000000..b9e86eb
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/FullOuterMapJoinOptimization.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.optimizer;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Properties;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
+import org.apache.hadoop.hive.ql.plan.MapJoinDesc;
+import org.apache.hadoop.hive.ql.plan.TableDesc;
+import org.apache.hadoop.hive.serde.serdeConstants;
+import org.apache.hadoop.hive.serde2.SerDeUtils;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
+
+import com.google.common.base.Joiner;
+
+/**
+ * FULL OUTER MapJoin planning.
+ */
+public class FullOuterMapJoinOptimization {
+
+ FullOuterMapJoinOptimization() {
+ }
+
+ public static void removeFilterMap(MapJoinDesc mapJoinDesc) throws SemanticException {
+ int[][] filterMaps = mapJoinDesc.getFilterMap();
+ if (filterMaps == null) {
+ return;
+ }
+ final byte posBigTable = (byte) mapJoinDesc.getPosBigTable();
+ final int numAliases = mapJoinDesc.getExprs().size();
+ List<TableDesc> valueFilteredTblDescs = mapJoinDesc.getValueFilteredTblDescs();
+ for (byte pos = 0; pos < numAliases; pos++) {
+ if (pos != posBigTable) {
+ int[] filterMap = filterMaps[pos];
+ TableDesc tableDesc = valueFilteredTblDescs.get(pos);
+ Properties properties = tableDesc.getProperties();
+ String columnNameProperty = properties.getProperty(serdeConstants.LIST_COLUMNS);
+ String columnNameDelimiter =
+ properties.containsKey(serdeConstants.COLUMN_NAME_DELIMITER) ?
+ properties.getProperty(serdeConstants.COLUMN_NAME_DELIMITER) :
+ String.valueOf(SerDeUtils.COMMA);
+
+ String columnTypeProperty = properties.getProperty(serdeConstants.LIST_COLUMN_TYPES);
+ List<String> columnNameList;
+ if (columnNameProperty.length() == 0) {
+ columnNameList = new ArrayList<String>();
+ } else {
+ columnNameList = Arrays.asList(columnNameProperty.split(columnNameDelimiter));
+ }
+ List<String> truncatedColumnNameList = columnNameList.subList(0, columnNameList.size() - 1);
+ String truncatedColumnNameProperty =
+ Joiner.on(columnNameDelimiter).join(truncatedColumnNameList);
+
+ List<TypeInfo> columnTypeList;
+ if (columnTypeProperty.length() == 0) {
+ columnTypeList = new ArrayList<TypeInfo>();
+ } else {
+ columnTypeList = TypeInfoUtils
+ .getTypeInfosFromTypeString(columnTypeProperty);
+ }
+ if (!columnTypeList.get(columnTypeList.size() - 1).equals(TypeInfoFactory.shortTypeInfo)) {
+ throw new SemanticException("Expecting filterTag smallint as last column type");
+ }
+ List<TypeInfo> truncatedColumnTypeList =
+ columnTypeList.subList(0, columnTypeList.size() - 1);
+ String truncatedColumnTypeProperty =
+ Joiner.on(",").join(truncatedColumnTypeList);
+
+ properties.setProperty(serdeConstants.LIST_COLUMNS, truncatedColumnNameProperty);
+ properties.setProperty(serdeConstants.LIST_COLUMN_TYPES, truncatedColumnTypeProperty);
+ }
+ }
+ mapJoinDesc.setFilterMap(null);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/a37827ec/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java
index 019372b..23186c6 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java
@@ -26,6 +26,7 @@ import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
+import java.util.Map.Entry;
import java.util.Set;
import java.util.Stack;
@@ -48,6 +49,7 @@ import org.apache.hadoop.hive.ql.exec.SMBMapJoinOperator;
import org.apache.hadoop.hive.ql.exec.ScriptOperator;
import org.apache.hadoop.hive.ql.exec.SelectOperator;
import org.apache.hadoop.hive.ql.exec.UnionOperator;
+import org.apache.hadoop.hive.ql.exec.persistence.MapJoinKey;
import org.apache.hadoop.hive.ql.lib.DefaultRuleDispatcher;
import org.apache.hadoop.hive.ql.lib.Dispatcher;
import org.apache.hadoop.hive.ql.lib.GraphWalker;
@@ -56,6 +58,7 @@ import org.apache.hadoop.hive.ql.lib.NodeProcessor;
import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx;
import org.apache.hadoop.hive.ql.lib.Rule;
import org.apache.hadoop.hive.ql.lib.RuleRegExp;
+import org.apache.hadoop.hive.ql.optimizer.physical.Vectorizer.EnabledOverride;
import org.apache.hadoop.hive.ql.parse.GenMapRedWalker;
import org.apache.hadoop.hive.ql.parse.ParseContext;
import org.apache.hadoop.hive.ql.parse.SemanticException;
@@ -75,6 +78,7 @@ import org.apache.hadoop.hive.ql.plan.SMBJoinDesc;
import org.apache.hadoop.hive.ql.plan.SelectDesc;
import org.apache.hadoop.hive.ql.plan.TableDesc;
import org.apache.hadoop.hive.serde.serdeConstants;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -336,6 +340,9 @@ public class MapJoinProcessor extends Transform {
// create the map-join operator
MapJoinOperator mapJoinOp = convertJoinOpMapJoinOp(conf,
op, leftInputJoin, baseSrc, mapAliases, mapJoinPos, noCheckOuterJoin);
+ if (mapJoinOp == null) {
+ return null;
+ }
// remove old parents
for (pos = 0; pos < newParentOps.size(); pos++) {
@@ -356,6 +363,213 @@ public class MapJoinProcessor extends Transform {
return mapJoinOp;
}
+ public static boolean onExpressionHasNullSafes(JoinDesc desc) {
+ boolean[] nullSafes = desc.getNullSafes();
+ if (nullSafes == null) {
+ return false;
+ }
+ for (boolean nullSafe : nullSafes) {
+ if (nullSafe) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private static boolean checkFullOuterMapJoinCompatible(HiveConf hiveConf,
+ JoinOperator joinOp) throws SemanticException {
+ JoinDesc joinDesc = joinOp.getConf();
+
+ // Make sure all key and value expressions are columns.
+ for (Entry<Byte, List<ExprNodeDesc>> mapEntry : joinDesc.getExprs().entrySet()) {
+ List<ExprNodeDesc> exprList = mapEntry.getValue();
+ for (ExprNodeDesc expr : exprList) {
+ if (!(expr instanceof ExprNodeColumnDesc)) {
+ LOG.debug("FULL OUTER MapJoin: only column expressions are supported " + expr.toString());
+ return false;
+ }
+ }
+ }
+
+ // Check for supported key data types.
+ Byte[] order = joinDesc.getTagOrder();
+ ExprNodeDesc[][] joinKeysArray = joinDesc.getJoinKeys();
+ for (int i = 0; i < order.length; i++) {
+ byte pos = order[i];
+ ExprNodeDesc[] keyExprs = joinKeysArray[pos];
+ for (ExprNodeDesc keyExpr : keyExprs) {
+ TypeInfo typeInfo = keyExpr.getTypeInfo();
+
+ // Verify we handle the key column types for an optimized table. This is the effectively
+ // the same check used in Tez HashTableLoader.
+ if (!MapJoinKey.isSupportedField(typeInfo)) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("FULL OUTER MapJoin not enabled: " +
+ " key type " + typeInfo.toString() + " not supported");
+ }
+ return false;
+ }
+ }
+ }
+
+ if (onExpressionHasNullSafes(joinDesc)) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("FULL OUTER MapJoin not enabled: " +
+ "nullsafe not supported");
+ }
+ return false;
+ }
+
+ boolean isVectorizationMapJoinNativeEnabled = HiveConf.getBoolVar(hiveConf,
+ HiveConf.ConfVars.HIVE_VECTORIZATION_MAPJOIN_NATIVE_ENABLED);
+ boolean isHybridHashJoin = HiveConf.getBoolVar(hiveConf,
+ HiveConf.ConfVars.HIVEUSEHYBRIDGRACEHASHJOIN);
+ if (isVectorizationMapJoinNativeEnabled && isHybridHashJoin) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("FULL OUTER MapJoin not enabled: " +
+ "Native Vector MapJoin and Hybrid Grace not supported");
+ }
+ return false;
+ }
+
+ if (joinDesc.getResidualFilterExprs() != null &&
+ joinDesc.getResidualFilterExprs().size() != 0) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("FULL OUTER MapJoin not enabled: " +
+ "non-equi joins not supported");
+ }
+ return false;
+ }
+
+ return true;
+ }
+
+ public static boolean precheckFullOuter(HiveConf hiveConf, JoinOperator joinOp)
+ throws SemanticException {
+ JoinDesc joinDesc = joinOp.getConf();
+ JoinCondDesc[] conds = joinDesc.getConds();
+
+ /*
+ * Are we even being asked to do a FULL OUTER JOIN?
+ */
+ boolean hasFullOuterJoin = false;
+ for (JoinCondDesc cond : conds) {
+ if (cond.getType() == JoinDesc.FULL_OUTER_JOIN) {
+ hasFullOuterJoin = true;
+ break;
+ }
+ }
+ if (!hasFullOuterJoin) {
+ return false;
+ }
+
+ if (conds.length > 1) {
+
+ // No multiple condition FULL OUTER MapJoin.
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("FULL OUTER MapJoin not enabled: multiple JOIN conditions not supported");
+ }
+ return false;
+ }
+
+ return true;
+ }
+
+ public static boolean isFullOuterMapEnabled(HiveConf hiveConf, JoinOperator joinOp)
+ throws SemanticException {
+
+ final String testMapJoinFullOuterOverrideString =
+ HiveConf.getVar(hiveConf,
+ HiveConf.ConfVars.HIVE_TEST_MAPJOINFULLOUTER_OVERRIDE);
+ EnabledOverride mapJoinFullOuterOverride =
+ EnabledOverride.nameMap.get(testMapJoinFullOuterOverrideString);
+
+ final boolean isEnabled =
+ HiveConf.getBoolVar(
+ hiveConf,
+ HiveConf.ConfVars.HIVEMAPJOINFULLOUTER);
+ switch (mapJoinFullOuterOverride) {
+ case NONE:
+ {
+ if (!isEnabled) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("FULL OUTER MapJoin not enabled: " +
+ HiveConf.ConfVars.HIVEMAPJOINFULLOUTER.varname + " is false");
+ }
+ return false;
+ }
+ }
+ break;
+ case DISABLE:
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("FULL OUTER MapJoin not enabled: " +
+ HiveConf.ConfVars.HIVE_TEST_MAPJOINFULLOUTER_OVERRIDE.varname + " is disable (" +
+ " " + HiveConf.ConfVars.HIVEMAPJOINFULLOUTER.varname + " is " + isEnabled + ")");
+ }
+ return false;
+ case ENABLE:
+
+ // Different parts of the code may rely on this being set...
+ HiveConf.setBoolVar(hiveConf,
+ HiveConf.ConfVars.HIVEMAPJOINFULLOUTER, true);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("FULL OUTER MapJoin is enabled: " +
+ HiveConf.ConfVars.HIVE_TEST_MAPJOINFULLOUTER_OVERRIDE.varname + " is enable (" +
+ " " + HiveConf.ConfVars.HIVEMAPJOINFULLOUTER.varname + " is " + isEnabled + ")");
+ }
+ break;
+ default:
+ throw new RuntimeException("Unexpected vectorization enabled override " +
+ mapJoinFullOuterOverride);
+ }
+
+ final String engine =
+ HiveConf.getVar(
+ hiveConf,
+ HiveConf.ConfVars.HIVE_EXECUTION_ENGINE);
+ final boolean isTezEngine = engine.equalsIgnoreCase("tez");
+ if (!isTezEngine) {
+
+ // Only Tez for now.
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("FULL OUTER MapJoin not enabled: Only Tez engine supported");
+ }
+ return false;
+ }
+
+ /*
+ * Optimized Hash Table (i.e. not old-style MR HashMap).
+ */
+ final boolean isOptimizedHashTableEnabled =
+ HiveConf.getBoolVar(
+ hiveConf,
+ HiveConf.ConfVars.HIVEMAPJOINUSEOPTIMIZEDTABLE);
+ if (!isOptimizedHashTableEnabled) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("FULL OUTER MapJoin not enabled: " +
+ HiveConf.ConfVars.HIVEMAPJOINUSEOPTIMIZEDTABLE.varname + " is false");
+ }
+ return false;
+ }
+
+ boolean isCompatibleFullOuterMapJoin = checkFullOuterMapJoinCompatible(hiveConf, joinOp);
+ if (!isCompatibleFullOuterMapJoin) {
+ return false;
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("FULL OUTER MapJoin enabled");
+ }
+ return true;
+ }
+
+ public static boolean isFullOuterEnabledForDynamicPartitionHashJoin(HiveConf hiveConf, JoinOperator joinOp)
+ throws SemanticException {
+ JoinDesc joinDesc = joinOp.getConf();
+
+ return true;
+ }
+
public static MapJoinOperator convertJoinOpMapJoinOp(HiveConf hconf,
JoinOperator op, boolean leftInputJoin, String[] baseSrc, List<String> mapAliases,
int mapJoinPos, boolean noCheckOuterJoin) throws SemanticException {
@@ -371,6 +585,9 @@ public class MapJoinProcessor extends Transform {
MapJoinDesc mapJoinDescriptor =
getMapJoinDesc(hconf, op, leftInputJoin, baseSrc, mapAliases,
mapJoinPos, noCheckOuterJoin, adjustParentsChildren);
+ if (mapJoinDescriptor == null) {
+ return null;
+ }
// reduce sink row resolver used to generate map join op
RowSchema outputRS = op.getSchema();
@@ -474,6 +691,9 @@ public class MapJoinProcessor extends Transform {
MapJoinOperator mapJoinOp = convertMapJoin(pctx.getConf(), op,
op.getConf().isLeftInputJoin(), op.getConf().getBaseSrc(),
op.getConf().getMapAliases(), mapJoinPos, noCheckOuterJoin, true);
+ if (mapJoinOp == null) {
+ return null;
+ }
// create a dummy select to select all columns
genSelectPlan(pctx, mapJoinOp);
@@ -502,8 +722,29 @@ public class MapJoinProcessor extends Transform {
* @return set of big table candidates
*/
public static Set<Integer> getBigTableCandidates(JoinCondDesc[] condns) {
+ return getBigTableCandidates(condns, /* isSupportFullOuter */ false);
+ }
+
+ public static Set<Integer> getBigTableCandidates(JoinCondDesc[] condns,
+ boolean isSupportFullOuter) {
+
Set<Integer> bigTableCandidates = new HashSet<Integer>();
+ if (condns.length == 1) {
+ JoinCondDesc condn = condns[0];
+ if (condn.getType() == JoinDesc.FULL_OUTER_JOIN) {
+
+ if (!isSupportFullOuter) {
+ return new HashSet<Integer>();
+ }
+
+ // FULL OUTER MapJoin must be a single condition.
+ bigTableCandidates.add(condn.getLeft());
+ bigTableCandidates.add(condn.getRight());
+ return bigTableCandidates;
+ }
+ }
+
boolean seenOuterJoin = false;
Set<Integer> seenPostitions = new HashSet<Integer>();
Set<Integer> leftPosListOfLastRightOuterJoin = new HashSet<Integer>();
@@ -512,17 +753,14 @@ public class MapJoinProcessor extends Transform {
boolean lastSeenRightOuterJoin = false;
for (JoinCondDesc condn : condns) {
int joinType = condn.getType();
+ if (joinType == JoinDesc.FULL_OUTER_JOIN) {
+ return new HashSet<Integer>();
+ }
+
seenPostitions.add(condn.getLeft());
seenPostitions.add(condn.getRight());
- if (joinType == JoinDesc.FULL_OUTER_JOIN) {
- // setting these 2 parameters here just in case that if the code got
- // changed in future, these 2 are not missing.
- seenOuterJoin = true;
- lastSeenRightOuterJoin = false;
- // empty set - cannot convert
- return new HashSet<Integer>();
- } else if (joinType == JoinDesc.LEFT_OUTER_JOIN
+ if (joinType == JoinDesc.LEFT_OUTER_JOIN
|| joinType == JoinDesc.LEFT_SEMI_JOIN) {
seenOuterJoin = true;
if(bigTableCandidates.size() == 0) {
@@ -564,7 +802,8 @@ public class MapJoinProcessor extends Transform {
* @throws SemanticException if given position is not in the big table candidates.
*/
public static int checkMapJoin(int mapJoinPos, JoinCondDesc[] condns) {
- Set<Integer> bigTableCandidates = MapJoinProcessor.getBigTableCandidates(condns);
+ Set<Integer> bigTableCandidates =
+ MapJoinProcessor.getBigTableCandidates(condns, /* isSupportFullOuter */ true);
// bigTableCandidates can never be null
if (!bigTableCandidates.contains(mapJoinPos)) {
@@ -1183,6 +1422,9 @@ public class MapJoinProcessor extends Transform {
}
List<ExprNodeDesc> keyCols = keyExprMap.get((byte) mapJoinPos);
+ if (keyCols == null) {
+ return null;
+ }
List<String> outputColumnNames = op.getConf().getOutputColumnNames();
TableDesc keyTableDesc =
http://git-wip-us.apache.org/repos/asf/hive/blob/a37827ec/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SparkMapJoinProcessor.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SparkMapJoinProcessor.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SparkMapJoinProcessor.java
index 16d088a..a5400d6 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SparkMapJoinProcessor.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SparkMapJoinProcessor.java
@@ -64,6 +64,9 @@ public class SparkMapJoinProcessor extends MapJoinProcessor {
MapJoinOperator mapJoinOp = convertJoinOpMapJoinOp(conf,
op, op.getConf().isLeftInputJoin(), op.getConf().getBaseSrc(),
op.getConf().getMapAliases(), bigTablePos, noCheckOuterJoin);
+ if (mapJoinOp == null) {
+ return null;
+ }
// 1. remove RS as parent for the big table branch
// 2. remove old join op from child set of all the RSs
http://git-wip-us.apache.org/repos/asf/hive/blob/a37827ec/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
index 48974f8..51b186c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
@@ -73,6 +73,9 @@ import org.apache.hadoop.hive.ql.exec.vector.mapjoin.VectorMapJoinLeftSemiString
import org.apache.hadoop.hive.ql.exec.vector.mapjoin.VectorMapJoinOuterLongOperator;
import org.apache.hadoop.hive.ql.exec.vector.mapjoin.VectorMapJoinOuterMultiKeyOperator;
import org.apache.hadoop.hive.ql.exec.vector.mapjoin.VectorMapJoinOuterStringOperator;
+import org.apache.hadoop.hive.ql.exec.vector.mapjoin.VectorMapJoinFullOuterLongOperator;
+import org.apache.hadoop.hive.ql.exec.vector.mapjoin.VectorMapJoinFullOuterMultiKeyOperator;
+import org.apache.hadoop.hive.ql.exec.vector.mapjoin.VectorMapJoinFullOuterStringOperator;
import org.apache.hadoop.hive.ql.exec.vector.ptf.VectorPTFOperator;
import org.apache.hadoop.hive.ql.exec.vector.udf.VectorUDFAdaptor;
import org.apache.hadoop.hive.ql.exec.vector.ColumnVector;
@@ -122,6 +125,7 @@ import org.apache.hadoop.hive.ql.plan.LimitDesc;
import org.apache.hadoop.hive.ql.plan.MapJoinDesc;
import org.apache.hadoop.hive.ql.plan.MapWork;
import org.apache.hadoop.hive.ql.plan.MapredWork;
+import org.apache.hadoop.hive.ql.plan.MergeJoinWork;
import org.apache.hadoop.hive.ql.plan.OperatorDesc;
import org.apache.hadoop.hive.ql.plan.PTFDesc;
import org.apache.hadoop.hive.ql.plan.SelectDesc;
@@ -289,15 +293,15 @@ public class Vectorizer implements PhysicalPlanResolver {
private HiveConf hiveConf;
- public static enum VectorizationEnabledOverride {
+ public enum EnabledOverride {
NONE,
DISABLE,
ENABLE;
- public final static Map<String,VectorizationEnabledOverride> nameMap =
- new HashMap<String,VectorizationEnabledOverride>();
+ public static final Map<String, EnabledOverride> nameMap =
+ new HashMap<String, EnabledOverride>();
static {
- for (VectorizationEnabledOverride vectorizationEnabledOverride : values()) {
+ for (EnabledOverride vectorizationEnabledOverride : values()) {
nameMap.put(
vectorizationEnabledOverride.name().toLowerCase(), vectorizationEnabledOverride);
}
@@ -305,7 +309,7 @@ public class Vectorizer implements PhysicalPlanResolver {
}
boolean isVectorizationEnabled;
- private VectorizationEnabledOverride vectorizationEnabledOverride;
+ private EnabledOverride vectorizationEnabledOverride;
boolean isTestForcedVectorizationEnable;
private boolean useVectorizedInputFileFormat;
@@ -725,10 +729,63 @@ public class Vectorizer implements PhysicalPlanResolver {
}
}
- private List<Operator<? extends OperatorDesc>> newOperatorList() {
+ private static List<Operator<? extends OperatorDesc>> newOperatorList() {
return new ArrayList<Operator<? extends OperatorDesc>>();
}
+ public static void debugDisplayJoinOperatorTree(Operator<? extends OperatorDesc> joinOperator,
+ String prefix) {
+ List<Operator<? extends OperatorDesc>> currentParentList = newOperatorList();
+ currentParentList.add(joinOperator);
+
+ int depth = 0;
+ do {
+ List<Operator<? extends OperatorDesc>> nextParentList = newOperatorList();
+
+ final int count = currentParentList.size();
+ for (int i = 0; i < count; i++) {
+ Operator<? extends OperatorDesc> parent = currentParentList.get(i);
+ System.out.println(prefix + " parent depth " + depth + " " +
+ parent.getClass().getSimpleName() + " " + parent.toString());
+
+ List<Operator<? extends OperatorDesc>> parentList = parent.getParentOperators();
+ if (parentList == null || parentList.size() == 0) {
+ continue;
+ }
+
+ nextParentList.addAll(parentList);
+ }
+
+ currentParentList = nextParentList;
+ depth--;
+ } while (currentParentList.size() > 0);
+
+ List<Operator<? extends OperatorDesc>> currentChildList = newOperatorList();
+ currentChildList.addAll(joinOperator.getChildOperators());
+
+ depth = 1;
+ do {
+ List<Operator<? extends OperatorDesc>> nextChildList = newOperatorList();
+
+ final int count = currentChildList.size();
+ for (int i = 0; i < count; i++) {
+ Operator<? extends OperatorDesc> child = currentChildList.get(i);
+ System.out.println(prefix + " child depth " + depth + " " +
+ child.getClass().getSimpleName() + " " + child.toString());
+
+ List<Operator<? extends OperatorDesc>> childList = child.getChildOperators();
+ if (childList == null || childList.size() == 0) {
+ continue;
+ }
+
+ nextChildList.addAll(childList);
+ }
+
+ currentChildList = nextChildList;
+ depth--;
+ } while (currentChildList.size() > 0);
+ }
+
private Operator<? extends OperatorDesc> validateAndVectorizeOperatorTree(
Operator<? extends OperatorDesc> nonVecRootOperator,
boolean isReduce, boolean isTezOrSpark,
@@ -973,7 +1030,15 @@ public class Vectorizer implements PhysicalPlanResolver {
if (isReduceVectorizationEnabled) {
convertReduceWork(reduceWork);
}
+
logReduceWorkExplainVectorization(reduceWork);
+ } else if (baseWork instanceof MergeJoinWork){
+ MergeJoinWork mergeJoinWork = (MergeJoinWork) baseWork;
+
+ // Always set the EXPLAIN conditions.
+ setMergeJoinWorkExplainConditions(mergeJoinWork);
+
+ logMergeJoinWorkExplainVectorization(mergeJoinWork);
}
}
} else if (currTask instanceof SparkTask) {
@@ -1029,6 +1094,11 @@ public class Vectorizer implements PhysicalPlanResolver {
HiveConf.getVar(hiveConf, HiveConf.ConfVars.HIVE_EXECUTION_ENGINE));
}
+ private void setMergeJoinWorkExplainConditions(MergeJoinWork mergeJoinWork) {
+
+ setExplainConditions(mergeJoinWork);
+ }
+
private boolean logExplainVectorization(BaseWork baseWork, String name) {
if (!baseWork.getVectorizationExamined()) {
@@ -1101,6 +1171,13 @@ public class Vectorizer implements PhysicalPlanResolver {
LOG.info("Reducer engine: " + reduceWork.getVectorReduceEngine());
}
+ private void logMergeJoinWorkExplainVectorization(MergeJoinWork mergeJoinWork) {
+
+ if (!logExplainVectorization(mergeJoinWork, "MergeJoin")) {
+ return;
+ }
+ }
+
private void convertMapWork(MapWork mapWork, boolean isTezOrSpark) throws SemanticException {
// We have to evaluate the input format to see if vectorization is enabled, so
@@ -2319,7 +2396,7 @@ public class Vectorizer implements PhysicalPlanResolver {
HiveConf.getVar(hiveConf,
HiveConf.ConfVars.HIVE_TEST_VECTORIZATION_ENABLED_OVERRIDE);
vectorizationEnabledOverride =
- VectorizationEnabledOverride.nameMap.get(vectorizationEnabledOverrideString);
+ EnabledOverride.nameMap.get(vectorizationEnabledOverrideString);
isVectorizationEnabled = HiveConf.getBoolVar(hiveConf,
HiveConf.ConfVars.HIVE_VECTORIZATION_ENABLED);
@@ -3264,7 +3341,7 @@ public class Vectorizer implements PhysicalPlanResolver {
HashTableImplementationType hashTableImplementationType = HashTableImplementationType.NONE;
HashTableKind hashTableKind = HashTableKind.NONE;
HashTableKeyType hashTableKeyType = HashTableKeyType.NONE;
- VectorMapJoinVariation vectorMapJoinVariation = VectorMapJoinVariation.NONE;
+ VectorMapJoinVariation vectorMapJoinVariation = null;
if (vectorDesc.getIsFastHashTableEnabled()) {
hashTableImplementationType = HashTableImplementationType.FAST;
@@ -3334,6 +3411,10 @@ public class Vectorizer implements PhysicalPlanResolver {
vectorMapJoinVariation = VectorMapJoinVariation.OUTER;
hashTableKind = HashTableKind.HASH_MAP;
break;
+ case JoinDesc.FULL_OUTER_JOIN:
+ vectorMapJoinVariation = VectorMapJoinVariation.FULL_OUTER;
+ hashTableKind = HashTableKind.HASH_MAP;
+ break;
case JoinDesc.LEFT_SEMI_JOIN:
vectorMapJoinVariation = VectorMapJoinVariation.LEFT_SEMI;
hashTableKind = HashTableKind.HASH_SET;
@@ -3363,6 +3444,9 @@ public class Vectorizer implements PhysicalPlanResolver {
case OUTER:
opClass = VectorMapJoinOuterLongOperator.class;
break;
+ case FULL_OUTER:
+ opClass = VectorMapJoinFullOuterLongOperator.class;
+ break;
default:
throw new HiveException("Unknown operator variation " + vectorMapJoinVariation);
}
@@ -3381,6 +3465,9 @@ public class Vectorizer implements PhysicalPlanResolver {
case OUTER:
opClass = VectorMapJoinOuterStringOperator.class;
break;
+ case FULL_OUTER:
+ opClass = VectorMapJoinFullOuterStringOperator.class;
+ break;
default:
throw new HiveException("Unknown operator variation " + vectorMapJoinVariation);
}
@@ -3399,6 +3486,9 @@ public class Vectorizer implements PhysicalPlanResolver {
case OUTER:
opClass = VectorMapJoinOuterMultiKeyOperator.class;
break;
+ case FULL_OUTER:
+ opClass = VectorMapJoinFullOuterMultiKeyOperator.class;
+ break;
default:
throw new HiveException("Unknown operator variation " + vectorMapJoinVariation);
}
@@ -3414,6 +3504,10 @@ public class Vectorizer implements PhysicalPlanResolver {
vectorDesc.setHashTableKind(hashTableKind);
vectorDesc.setHashTableKeyType(hashTableKeyType);
vectorDesc.setVectorMapJoinVariation(vectorMapJoinVariation);
+ if (vectorMapJoinVariation == VectorMapJoinVariation.FULL_OUTER) {
+
+ vectorDesc.setIsFullOuter(true);
+ }
vectorDesc.setMinMaxEnabled(minMaxEnabled);
vectorDesc.setVectorMapJoinInfo(vectorMapJoinInfo);
@@ -3526,6 +3620,8 @@ public class Vectorizer implements PhysicalPlanResolver {
/*
* Similarly, we need a mapping since a value expression can be a calculation and the value
* will go into a scratch column.
+ *
+ * Value expressions include keys? YES.
*/
int[] bigTableValueColumnMap = new int[allBigTableValueExpressions.length];
String[] bigTableValueColumnNames = new String[allBigTableValueExpressions.length];
@@ -3565,18 +3661,24 @@ public class Vectorizer implements PhysicalPlanResolver {
vectorDesc.setAllBigTableValueExpressions(allBigTableValueExpressions);
/*
- * Small table information.
+ * Column mapping.
*/
- VectorColumnOutputMapping bigTableRetainedMapping =
- new VectorColumnOutputMapping("Big Table Retained Mapping");
+ VectorColumnOutputMapping bigTableRetainMapping =
+ new VectorColumnOutputMapping("Big Table Retain Mapping");
- VectorColumnOutputMapping bigTableOuterKeyMapping =
- new VectorColumnOutputMapping("Big Table Outer Key Mapping");
+ VectorColumnOutputMapping nonOuterSmallTableKeyMapping =
+ new VectorColumnOutputMapping("Non Outer Small Table Key Key Mapping");
+
+ VectorColumnOutputMapping outerSmallTableKeyMapping =
+ new VectorColumnOutputMapping("Outer Small Table Key Mapping");
+
+ VectorColumnSourceMapping fullOuterSmallTableKeyMapping =
+ new VectorColumnSourceMapping("Full Outer Small Table Key Mapping");
// The order of the fields in the LazyBinary small table value must be used, so
// we use the source ordering flavor for the mapping.
- VectorColumnSourceMapping smallTableMapping =
- new VectorColumnSourceMapping("Small Table Mapping");
+ VectorColumnSourceMapping smallTableValueMapping =
+ new VectorColumnSourceMapping("Small Table Value Mapping");
Byte[] order = desc.getTagOrder();
Byte posSingleVectorMapJoinSmallTable = (order[0] == posBigTable ? order[1] : order[0]);
@@ -3586,7 +3688,6 @@ public class Vectorizer implements PhysicalPlanResolver {
* Gather up big and small table output result information from the MapJoinDesc.
*/
List<Integer> bigTableRetainList = desc.getRetainList().get(posBigTable);
- int bigTableRetainSize = bigTableRetainList.size();
int[] smallTableIndices;
int smallTableIndicesSize;
@@ -3623,6 +3724,8 @@ public class Vectorizer implements PhysicalPlanResolver {
VectorColumnSourceMapping projectionMapping = new VectorColumnSourceMapping("Projection Mapping");
int nextOutputColumn = (order[0] == posBigTable ? 0 : smallTableResultSize);
+
+ final int bigTableRetainSize = bigTableRetainList.size();
for (int i = 0; i < bigTableRetainSize; i++) {
// Since bigTableValueExpressions may do a calculation and produce a scratch column, we
@@ -3636,9 +3739,10 @@ public class Vectorizer implements PhysicalPlanResolver {
projectionMapping.add(nextOutputColumn, batchColumnIndex, typeInfo);
// Collect columns we copy from the big table batch to the overflow batch.
- if (!bigTableRetainedMapping.containsOutputColumn(batchColumnIndex)) {
+ if (!bigTableRetainMapping.containsOutputColumn(batchColumnIndex)) {
+
// Tolerate repeated use of a big table column.
- bigTableRetainedMapping.add(batchColumnIndex, batchColumnIndex, typeInfo);
+ bigTableRetainMapping.add(batchColumnIndex, batchColumnIndex, typeInfo);
}
nextOutputColumn++;
@@ -3655,10 +3759,8 @@ public class Vectorizer implements PhysicalPlanResolver {
nextOutputColumn = firstSmallTableOutputColumn;
// Small table indices has more information (i.e. keys) than retain, so use it if it exists...
- String[] bigTableRetainedNames;
if (smallTableIndicesSize > 0) {
smallTableOutputCount = smallTableIndicesSize;
- bigTableRetainedNames = new String[smallTableOutputCount];
for (int i = 0; i < smallTableIndicesSize; i++) {
if (smallTableIndices[i] >= 0) {
@@ -3670,8 +3772,7 @@ public class Vectorizer implements PhysicalPlanResolver {
// Since bigTableKeyExpressions may do a calculation and produce a scratch column, we
// need to map the right column.
- int batchKeyColumn = bigTableKeyColumnMap[keyIndex];
- bigTableRetainedNames[i] = bigTableKeyColumnNames[keyIndex];
+ int bigTableKeyColumn = bigTableKeyColumnMap[keyIndex];
TypeInfo typeInfo = bigTableKeyTypeInfos[keyIndex];
if (!isOuterJoin) {
@@ -3679,25 +3780,30 @@ public class Vectorizer implements PhysicalPlanResolver {
// Optimize inner join keys of small table results.
// Project the big table key into the small table result "area".
- projectionMapping.add(nextOutputColumn, batchKeyColumn, typeInfo);
+ projectionMapping.add(nextOutputColumn, bigTableKeyColumn, typeInfo);
- if (!bigTableRetainedMapping.containsOutputColumn(batchKeyColumn)) {
- // If necessary, copy the big table key into the overflow batch's small table
- // result "area".
- bigTableRetainedMapping.add(batchKeyColumn, batchKeyColumn, typeInfo);
+ if (!bigTableRetainMapping.containsOutputColumn(bigTableKeyColumn)) {
+
+ // When the Big Key is not retained in the output result, we do need to copy the
+ // Big Table key into the overflow batch so the projection of it (Big Table key) to
+ // the Small Table key will work properly...
+ //
+ nonOuterSmallTableKeyMapping.add(bigTableKeyColumn, bigTableKeyColumn, typeInfo);
}
} else {
- // For outer joins, since the small table key can be null when there is no match,
+ // For outer joins, since the small table key can be null when there for NOMATCH,
// we must have a physical (scratch) column for those keys. We cannot use the
- // projection optimization used by inner joins above.
+ // projection optimization used by non-[FULL} OUTER joins above.
int scratchColumn = vContext.allocateScratchColumn(typeInfo);
projectionMapping.add(nextOutputColumn, scratchColumn, typeInfo);
- bigTableRetainedMapping.add(batchKeyColumn, scratchColumn, typeInfo);
+ outerSmallTableKeyMapping.add(bigTableKeyColumn, scratchColumn, typeInfo);
- bigTableOuterKeyMapping.add(batchKeyColumn, scratchColumn, typeInfo);
+ // For FULL OUTER MapJoin, we need to be able to deserialize a Small Table key
+ // into the output result.
+ fullOuterSmallTableKeyMapping.add(keyIndex, scratchColumn, typeInfo);
}
} else {
@@ -3711,21 +3817,18 @@ public class Vectorizer implements PhysicalPlanResolver {
smallTableExprVectorizes = false;
}
- bigTableRetainedNames[i] = smallTableExprNode.toString();
-
TypeInfo typeInfo = smallTableExprNode.getTypeInfo();
// Make a new big table scratch column for the small table value.
int scratchColumn = vContext.allocateScratchColumn(typeInfo);
projectionMapping.add(nextOutputColumn, scratchColumn, typeInfo);
- smallTableMapping.add(smallTableValueIndex, scratchColumn, typeInfo);
+ smallTableValueMapping.add(smallTableValueIndex, scratchColumn, typeInfo);
}
nextOutputColumn++;
}
} else if (smallTableRetainSize > 0) {
smallTableOutputCount = smallTableRetainSize;
- bigTableRetainedNames = new String[smallTableOutputCount];
// Only small table values appear in join output result.
@@ -3738,21 +3841,24 @@ public class Vectorizer implements PhysicalPlanResolver {
smallTableExprVectorizes = false;
}
- bigTableRetainedNames[i] = smallTableExprNode.toString();
-
// Make a new big table scratch column for the small table value.
TypeInfo typeInfo = smallTableExprNode.getTypeInfo();
int scratchColumn = vContext.allocateScratchColumn(typeInfo);
projectionMapping.add(nextOutputColumn, scratchColumn, typeInfo);
- smallTableMapping.add(smallTableValueIndex, scratchColumn, typeInfo);
+ smallTableValueMapping.add(smallTableValueIndex, scratchColumn, typeInfo);
nextOutputColumn++;
}
- } else {
- bigTableRetainedNames = new String[0];
}
+ Map<Byte, List<ExprNodeDesc>> filterExpressions = desc.getFilters();
+ VectorExpression[] bigTableFilterExpressions =
+ vContext.getVectorExpressions(
+ filterExpressions.get(posBigTable),
+ VectorExpressionDescriptor.Mode.FILTER);
+ vectorMapJoinInfo.setBigTableFilterExpressions(bigTableFilterExpressions);
+
boolean useOptimizedTable =
HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVEMAPJOINUSEOPTIMIZEDTABLE);
@@ -3808,15 +3914,23 @@ public class Vectorizer implements PhysicalPlanResolver {
// Convert dynamic arrays and maps to simple arrays.
- bigTableRetainedMapping.finalize();
+ bigTableRetainMapping.finalize();
+ vectorMapJoinInfo.setBigTableRetainColumnMap(bigTableRetainMapping.getOutputColumns());
+ vectorMapJoinInfo.setBigTableRetainTypeInfos(bigTableRetainMapping.getTypeInfos());
+
+ nonOuterSmallTableKeyMapping.finalize();
+ vectorMapJoinInfo.setNonOuterSmallTableKeyColumnMap(nonOuterSmallTableKeyMapping.getOutputColumns());
+ vectorMapJoinInfo.setNonOuterSmallTableKeyTypeInfos(nonOuterSmallTableKeyMapping.getTypeInfos());
+
+ outerSmallTableKeyMapping.finalize();
+ fullOuterSmallTableKeyMapping.finalize();
- bigTableOuterKeyMapping.finalize();
+ vectorMapJoinInfo.setOuterSmallTableKeyMapping(outerSmallTableKeyMapping);
+ vectorMapJoinInfo.setFullOuterSmallTableKeyMapping(fullOuterSmallTableKeyMapping);
- smallTableMapping.finalize();
+ smallTableValueMapping.finalize();
- vectorMapJoinInfo.setBigTableRetainedMapping(bigTableRetainedMapping);
- vectorMapJoinInfo.setBigTableOuterKeyMapping(bigTableOuterKeyMapping);
- vectorMapJoinInfo.setSmallTableMapping(smallTableMapping);
+ vectorMapJoinInfo.setSmallTableValueMapping(smallTableValueMapping);
projectionMapping.finalize();
http://git-wip-us.apache.org/repos/asf/hive/blob/a37827ec/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkMapJoinOptimizer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkMapJoinOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkMapJoinOptimizer.java
index 689c888..ad6db21 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkMapJoinOptimizer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkMapJoinOptimizer.java
@@ -459,6 +459,9 @@ public class SparkMapJoinOptimizer implements NodeProcessor {
MapJoinProcessor.convertJoinOpMapJoinOp(context.getConf(), joinOp,
joinOp.getConf().isLeftInputJoin(), joinOp.getConf().getBaseSrc(),
joinOp.getConf().getMapAliases(), bigTablePosition, true);
+ if (mapJoinOp == null) {
+ return null;
+ }
Operator<? extends OperatorDesc> parentBigTableOp =
mapJoinOp.getParentOperators().get(bigTablePosition);
http://git-wip-us.apache.org/repos/asf/hive/blob/a37827ec/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainConfiguration.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainConfiguration.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainConfiguration.java
index 2f12bda..a92502e 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainConfiguration.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainConfiguration.java
@@ -49,6 +49,7 @@ public class ExplainConfiguration {
private VectorizationDetailLevel vectorizationDetailLevel = VectorizationDetailLevel.SUMMARY;
private boolean locks = false;
private boolean ast = false;
+ private boolean debug = false;
private Path explainRootPath;
private Map<String, Long> opIdToRuntimeNumRows;
@@ -139,6 +140,14 @@ public class ExplainConfiguration {
this.vectorizationDetailLevel = vectorizationDetailLevel;
}
+ public boolean isDebug() {
+ return debug;
+ }
+
+ public void setDebug(boolean debug) {
+ this.debug = debug;
+ }
+
public Path getExplainRootPath() {
return explainRootPath;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/a37827ec/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java
index b17efff..428a3c9 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java
@@ -116,6 +116,8 @@ public class ExplainSemanticAnalyzer extends BaseSemanticAnalyzer {
config.setLocks(true);
} else if (explainOptions == HiveParser.KW_AST){
config.setAst(true);
+ } else if (explainOptions == HiveParser.KW_DEBUG) {
+ config.setDebug(true);
} else {
// UNDONE: UNKNOWN OPTION?
}
http://git-wip-us.apache.org/repos/asf/hive/blob/a37827ec/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g b/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g
index 807f6f5..3caa51f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g
@@ -189,6 +189,7 @@ KW_FILE: 'FILE';
KW_JAR: 'JAR';
KW_EXPLAIN: 'EXPLAIN';
KW_EXTENDED: 'EXTENDED';
+KW_DEBUG: 'DEBUG';
KW_FORMATTED: 'FORMATTED';
KW_DEPENDENCY: 'DEPENDENCY';
KW_LOGICAL: 'LOGICAL';
http://git-wip-us.apache.org/repos/asf/hive/blob/a37827ec/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g b/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g
index 2cfcdcc..48f7303 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g
@@ -569,6 +569,7 @@ import org.apache.hadoop.hive.conf.HiveConf;
xlateMap.put("KW_FUNCTION", "FUNCTION");
xlateMap.put("KW_EXPLAIN", "EXPLAIN");
xlateMap.put("KW_EXTENDED", "EXTENDED");
+ xlateMap.put("KW_DEBUG", "DEBUG");
xlateMap.put("KW_SERDE", "SERDE");
xlateMap.put("KW_WITH", "WITH");
xlateMap.put("KW_SERDEPROPERTIES", "SERDEPROPERTIES");
@@ -810,6 +811,7 @@ explainOption
| KW_LOCKS
| KW_AST
| (KW_VECTORIZATION vectorizationOnly? vectorizatonDetail?)
+ | KW_DEBUG
;
vectorizationOnly
http://git-wip-us.apache.org/repos/asf/hive/blob/a37827ec/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g b/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g
index f327a12..f9c97e0 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g
@@ -831,6 +831,7 @@ nonReserved
| KW_OPERATOR
| KW_EXPRESSION
| KW_DETAIL
+ | KW_DEBUG
| KW_WAIT
| KW_ZONE
| KW_TIMESTAMPTZ
http://git-wip-us.apache.org/repos/asf/hive/blob/a37827ec/ql/src/java/org/apache/hadoop/hive/ql/plan/Explain.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/Explain.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/Explain.java
index da30243..030bb61 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/Explain.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/Explain.java
@@ -28,7 +28,7 @@ import java.lang.annotation.RetentionPolicy;
@Retention(RetentionPolicy.RUNTIME)
public @interface Explain {
public enum Level {
- USER, DEFAULT, EXTENDED;
+ USER, DEFAULT, EXTENDED, DEBUG;
public boolean in(Level[] levels) {
for (Level level : levels) {
if (level.equals(this)) {
http://git-wip-us.apache.org/repos/asf/hive/blob/a37827ec/ql/src/java/org/apache/hadoop/hive/ql/plan/ExplainWork.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ExplainWork.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/ExplainWork.java
index 0654f57..01da4d5 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/ExplainWork.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/ExplainWork.java
@@ -165,6 +165,10 @@ public class ExplainWork implements Serializable {
return config.getVectorizationDetailLevel();
}
+ public boolean isDebug() {
+ return config.isDebug();
+ }
+
public ParseContext getParseContext() {
return pCtx;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/a37827ec/ql/src/java/org/apache/hadoop/hive/ql/plan/JoinCondDesc.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/JoinCondDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/JoinCondDesc.java
index 3820d57..0eb03e9 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/JoinCondDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/JoinCondDesc.java
@@ -126,7 +126,7 @@ public class JoinCondDesc implements Serializable {
sb.append("Inner Join ");
break;
case JoinDesc.FULL_OUTER_JOIN:
- sb.append("Outer Join ");
+ sb.append("Full Outer Join ");
break;
case JoinDesc.LEFT_OUTER_JOIN:
sb.append("Left Outer Join ");
http://git-wip-us.apache.org/repos/asf/hive/blob/a37827ec/ql/src/java/org/apache/hadoop/hive/ql/plan/JoinDesc.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/JoinDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/JoinDesc.java
index 4313a6b..2c93c2a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/JoinDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/JoinDesc.java
@@ -291,7 +291,7 @@ public class JoinDesc extends AbstractOperatorDesc {
first = false;
sb.append("{");
- sb.append(expr.getExprString());
+ sb.append(expr == null ? "NULL" : expr.getExprString());
sb.append("}");
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/a37827ec/ql/src/java/org/apache/hadoop/hive/ql/plan/MapJoinDesc.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/MapJoinDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/MapJoinDesc.java
index dc4f085..8ba5101 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/MapJoinDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/MapJoinDesc.java
@@ -33,10 +33,16 @@ import java.util.Set;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.exec.MemoryMonitorInfo;
+import org.apache.hadoop.hive.ql.exec.persistence.MapJoinObjectSerDeContext;
+import org.apache.hadoop.hive.ql.optimizer.signature.Signature;
import org.apache.hadoop.hive.ql.plan.Explain.Level;
import org.apache.hadoop.hive.ql.plan.Explain.Vectorization;
import org.apache.hadoop.hive.ql.plan.VectorMapJoinDesc.HashTableImplementationType;
import org.apache.hadoop.hive.ql.plan.VectorMapJoinDesc.VectorMapJoinVariation;
+import org.apache.hadoop.hive.serde2.AbstractSerDe;
+import org.apache.hadoop.hive.serde2.SerDeException;
+import org.apache.hadoop.hive.serde2.SerDeUtils;
+import org.apache.hive.common.util.ReflectionUtil;
/**
* Map Join operator Descriptor implementation.
@@ -91,6 +97,7 @@ public class MapJoinDesc extends JoinDesc implements Serializable {
this.keys = clone.keys;
this.keyTblDesc = clone.keyTblDesc;
this.valueTblDescs = clone.valueTblDescs;
+ this.valueFilteredTblDescs = clone.valueFilteredTblDescs;
this.posBigTable = clone.posBigTable;
this.valueIndices = clone.valueIndices;
this.retainList = clone.retainList;
@@ -208,6 +215,16 @@ public class MapJoinDesc extends JoinDesc implements Serializable {
this.dumpFilePrefix = dumpFilePrefix;
}
+ // NOTE: Debugging only.
+ @Explain(displayName = "keyExpressions", explainLevels = { Level.DEBUG })
+ public Map<Byte, String> getKeyExpressionString() {
+ Map<Byte, String> keyMap = new LinkedHashMap<Byte, String>();
+ for (Map.Entry<Byte, List<ExprNodeDesc>> k: getKeys().entrySet()) {
+ keyMap.put(k.getKey(), k.getValue().toString());
+ }
+ return keyMap;
+ }
+
/**
* @return the keys in string form
*/
@@ -292,6 +309,60 @@ public class MapJoinDesc extends JoinDesc implements Serializable {
return valueTblDescs;
}
+ // NOTE: Debugging only.
+ @Explain(displayName = "keyContext", explainLevels = { Level.DEBUG })
+ public String getDebugKeyContext() {
+ MapJoinObjectSerDeContext keyContext;
+ try {
+ AbstractSerDe keySerde =
+ (AbstractSerDe) ReflectionUtil.newInstance(
+ keyTblDesc.getDeserializerClass(), null);
+ SerDeUtils.initializeSerDe(keySerde, null, keyTblDesc.getProperties(), null);
+ keyContext = new MapJoinObjectSerDeContext(keySerde, false);
+ } catch (SerDeException e) {
+ return null;
+ }
+ return keyContext.stringify();
+ }
+
+ private boolean hasFilter(int alias, int[][] filterMaps) {
+ return filterMaps != null && filterMaps[alias] != null;
+ }
+
+ // NOTE: Debugging only.
+ @Explain(displayName = "valueContexts", explainLevels = { Level.DEBUG })
+ public String getDebugValueContext() {
+ List<String> valueContextStringList = new ArrayList<String>();
+ try {
+ boolean noOuterJoin = getNoOuterJoin();
+ // Order in which the results should be output.
+ Byte[] order = getTagOrder();
+ int[][] filterMaps = getFilterMap();
+
+ for (int pos = 0; pos < order.length; pos++) {
+ if (pos == posBigTable) {
+ continue;
+ }
+ TableDesc valueTableDesc;
+ if (noOuterJoin) {
+ valueTableDesc = getValueTblDescs().get(pos);
+ } else {
+ valueTableDesc = getValueFilteredTblDescs().get(pos);
+ }
+ AbstractSerDe valueSerDe =
+ (AbstractSerDe) ReflectionUtil.newInstance(
+ valueTableDesc.getDeserializerClass(), null);
+ SerDeUtils.initializeSerDe(valueSerDe, null, valueTableDesc.getProperties(), null);
+ MapJoinObjectSerDeContext valueContext =
+ new MapJoinObjectSerDeContext(valueSerDe, hasFilter(pos, filterMaps));
+ valueContextStringList.add(pos + ":" + valueContext.stringify());
+ }
+ } catch (SerDeException e) {
+ return null;
+ }
+ return valueContextStringList.toString();
+ }
+
/**
* @param valueTblDescs
* the valueTblDescs to set
@@ -378,6 +449,8 @@ public class MapJoinDesc extends JoinDesc implements Serializable {
return genJoinKeys;
}
+ @Explain(displayName = "DynamicPartitionHashJoin", explainLevels = { Level.USER, Level.DEFAULT,
+ Level.EXTENDED }, displayOnlyOnTrue = true)
public boolean isDynamicPartitionHashJoin() {
return isDynamicPartitionHashJoin;
}
@@ -386,6 +459,25 @@ public class MapJoinDesc extends JoinDesc implements Serializable {
this.isDynamicPartitionHashJoin = isDistributedHashJoin;
}
+ // NOTE: Debugging only.
+ @Explain(displayName = "outer filter mappings", explainLevels = { Level.DEBUG })
+ public String getDebugOuterFilterMapString() {
+ if (conds.length != 1) {
+ return null;
+ }
+ JoinCondDesc cond = conds[0];
+ if (cond.getType() != JoinDesc.FULL_OUTER_JOIN &&
+ cond.getType() != JoinDesc.LEFT_OUTER_JOIN &&
+ cond.getType() != JoinDesc.RIGHT_OUTER_JOIN) {
+ return null;
+ }
+ int[][] fm = getFilterMap();
+ if (fm == null) {
+ return null;
+ }
+ return Arrays.deepToString(fm);
+ }
+
// Use LinkedHashSet to give predictable display order.
private static final Set<String> vectorizableMapJoinNativeEngines =
new LinkedHashSet<String>(Arrays.asList("tez", "spark"));
@@ -401,7 +493,9 @@ public class MapJoinDesc extends JoinDesc implements Serializable {
public MapJoinOperatorExplainVectorization(MapJoinDesc mapJoinDesc,
VectorMapJoinDesc vectorMapJoinDesc) {
// VectorMapJoinOperator is not native vectorized.
- super(vectorMapJoinDesc, vectorMapJoinDesc.getHashTableImplementationType() != HashTableImplementationType.NONE);
+ super(
+ vectorMapJoinDesc,
+ vectorMapJoinDesc.getHashTableImplementationType() != HashTableImplementationType.NONE);
this.mapJoinDesc = mapJoinDesc;
this.vectorMapJoinDesc = vectorMapJoinDesc;
vectorMapJoinInfo =
@@ -414,7 +508,8 @@ public class MapJoinDesc extends JoinDesc implements Serializable {
String engine = vectorMapJoinDesc.getEngine();
String engineInSupportedCondName =
- HiveConf.ConfVars.HIVE_EXECUTION_ENGINE.varname + " " + engine + " IN " + vectorizableMapJoinNativeEngines;
+ HiveConf.ConfVars.HIVE_EXECUTION_ENGINE.varname + " " +
+ engine + " IN " + vectorizableMapJoinNativeEngines;
boolean engineInSupported = vectorizableMapJoinNativeEngines.contains(engine);
boolean isFastHashTableEnabled = vectorMapJoinDesc.getIsFastHashTableEnabled();
@@ -469,7 +564,8 @@ public class MapJoinDesc extends JoinDesc implements Serializable {
return conditions;
}
- @Explain(vectorization = Vectorization.OPERATOR, displayName = "nativeConditionsMet", explainLevels = { Level.DEFAULT, Level.EXTENDED })
+ @Explain(vectorization = Vectorization.OPERATOR, displayName = "nativeConditionsMet",
+ explainLevels = { Level.DEFAULT, Level.EXTENDED })
public List<String> getNativeConditionsMet() {
if (nativeConditions == null) {
nativeConditions = createNativeConditions();
@@ -477,7 +573,8 @@ public class MapJoinDesc extends JoinDesc implements Serializable {
return VectorizationCondition.getConditionsMet(nativeConditions);
}
- @Explain(vectorization = Vectorization.OPERATOR, displayName = "nativeConditionsNotMet", explainLevels = { Level.DEFAULT, Level.EXTENDED })
+ @Explain(vectorization = Vectorization.OPERATOR, displayName = "nativeConditionsNotMet",
+ explainLevels = { Level.DEFAULT, Level.EXTENDED })
public List<String> getNativeConditionsNotMet() {
if (nativeConditions == null) {
nativeConditions = createNativeConditions();
@@ -485,7 +582,8 @@ public class MapJoinDesc extends JoinDesc implements Serializable {
return VectorizationCondition.getConditionsNotMet(nativeConditions);
}
- @Explain(vectorization = Vectorization.EXPRESSION, displayName = "bigTableKeyExpressions", explainLevels = { Level.DEFAULT, Level.EXTENDED })
+ @Explain(vectorization = Vectorization.OPERATOR, displayName = "bigTableKeyExpressions",
+ explainLevels = { Level.DEFAULT, Level.EXTENDED })
public List<String> getBigTableKeyExpressions() {
return vectorExpressionsToStringList(
isNative ?
@@ -493,8 +591,18 @@ public class MapJoinDesc extends JoinDesc implements Serializable {
vectorMapJoinDesc.getAllBigTableKeyExpressions());
}
- @Explain(vectorization = Vectorization.DETAIL, displayName = "bigTableKeyColumnNums", explainLevels = { Level.DEFAULT, Level.EXTENDED })
- public String getBigTableKeyColumnNums() {
+ @Explain(vectorization = Vectorization.EXPRESSION, displayName = "hashTableImplementationType",
+ explainLevels = { Level.DEFAULT, Level.EXTENDED })
+ public String hashTableImplementationType() {
+ if (!isNative) {
+ return null;
+ }
+ return vectorMapJoinDesc.getHashTableImplementationType().name();
+ }
+
+ @Explain(vectorization = Vectorization.DETAIL, displayName = "bigTableKeyColumns",
+ explainLevels = { Level.DEFAULT, Level.EXTENDED })
+ public List<String> getBigTableKeyColumns() {
if (!isNative) {
return null;
}
@@ -502,10 +610,13 @@ public class MapJoinDesc extends JoinDesc implements Serializable {
if (bigTableKeyColumnMap.length == 0) {
return null;
}
- return Arrays.toString(bigTableKeyColumnMap);
+ return outputColumnsAndTypesToStringList(
+ vectorMapJoinInfo.getBigTableKeyColumnMap(),
+ vectorMapJoinInfo.getBigTableKeyTypeInfos());
}
- @Explain(vectorization = Vectorization.EXPRESSION, displayName = "bigTableValueExpressions", explainLevels = { Level.DEFAULT, Level.EXTENDED })
+ @Explain(vectorization = Vectorization.EXPRESSION, displayName = "bigTableValueExpressions",
+ explainLevels = { Level.DEFAULT, Level.EXTENDED })
public List<String> getBigTableValueExpressions() {
return vectorExpressionsToStringList(
isNative ?
@@ -513,8 +624,18 @@ public class MapJoinDesc extends JoinDesc implements Serializable {
vectorMapJoinDesc.getAllBigTableValueExpressions());
}
- @Explain(vectorization = Vectorization.DETAIL, displayName = "bigTableValueColumnNums", explainLevels = { Level.DEFAULT, Level.EXTENDED })
- public String getBigTableValueColumnNums() {
+ @Explain(vectorization = Vectorization.EXPRESSION, displayName = "bigTableFilterExpressions",
+ explainLevels = { Level.DEFAULT, Level.EXTENDED })
+ public List<String> getBigTableFilterExpressions() {
+ if (!isNative) {
+ return null;
+ }
+ return vectorExpressionsToStringList(vectorMapJoinInfo.getBigTableFilterExpressions());
+ }
+
+ @Explain(vectorization = Vectorization.DETAIL, displayName = "bigTableValueColumns",
+ explainLevels = { Level.DEFAULT, Level.EXTENDED })
+ public List<String> getBigTableValueColumns() {
if (!isNative) {
return null;
}
@@ -522,48 +643,78 @@ public class MapJoinDesc extends JoinDesc implements Serializable {
if (bigTableValueColumnMap.length == 0) {
return null;
}
- return Arrays.toString(bigTableValueColumnMap);
+ return outputColumnsAndTypesToStringList(
+ vectorMapJoinInfo.getBigTableValueColumnMap(),
+ vectorMapJoinInfo.getBigTableValueTypeInfos());
}
- @Explain(vectorization = Vectorization.DETAIL, displayName = "smallTableMapping", explainLevels = { Level.DEFAULT, Level.EXTENDED })
- public String getSmallTableColumns() {
+ @Explain(vectorization = Vectorization.DETAIL, displayName = "smallTableValueMapping",
+ explainLevels = { Level.DEFAULT, Level.EXTENDED })
+ public List<String> getSmallTableColumns() {
if (!isNative) {
return null;
}
- return outputColumnsToStringList(vectorMapJoinInfo.getSmallTableMapping());
+ return outputColumnsAndTypesToStringList(vectorMapJoinInfo.getSmallTableValueMapping());
}
- @Explain(vectorization = Vectorization.DETAIL, displayName = "projectedOutputColumnNums", explainLevels = { Level.DEFAULT, Level.EXTENDED })
- public String getProjectedOutputColumnNums() {
+ @Explain(vectorization = Vectorization.DETAIL, displayName = "projectedOutput",
+ explainLevels = { Level.DEFAULT, Level.EXTENDED })
+ public List<String> getProjectedOutputColumnNums() {
if (!isNative) {
return null;
}
- return outputColumnsToStringList(vectorMapJoinInfo.getProjectionMapping());
+ return outputColumnsAndTypesToStringList(vectorMapJoinInfo.getProjectionMapping());
}
- @Explain(vectorization = Vectorization.DETAIL, displayName = "bigTableOuterKeyMapping", explainLevels = { Level.DEFAULT, Level.EXTENDED })
- public List<String> getBigTableOuterKey() {
- if (!isNative || vectorMapJoinDesc.getVectorMapJoinVariation() != VectorMapJoinVariation.OUTER) {
+ @Explain(vectorization = Vectorization.DETAIL, displayName = "bigTableRetainColumnNums",
+ explainLevels = { Level.DEFAULT, Level.EXTENDED })
+ public String getBigTableRetainedColumnNums() {
+ if (!isNative) {
return null;
}
- return columnMappingToStringList(vectorMapJoinInfo.getBigTableOuterKeyMapping());
+ return Arrays.toString(vectorMapJoinInfo.getBigTableRetainColumnMap());
}
- @Explain(vectorization = Vectorization.DETAIL, displayName = "bigTableRetainedColumnNums", explainLevels = { Level.DEFAULT, Level.EXTENDED })
- public String getBigTableRetainedColumnNums() {
- if (!isNative) {
+ @Explain(vectorization = Vectorization.DETAIL, displayName = "nonOuterSmallTableKeyMapping",
+ explainLevels = { Level.DEFAULT, Level.EXTENDED })
+ public String getNonOuterSmallTableKeyMapping() {
+ if (!isNative ||
+ (vectorMapJoinDesc.getVectorMapJoinVariation() == VectorMapJoinVariation.OUTER ||
+ vectorMapJoinDesc.getVectorMapJoinVariation() == VectorMapJoinVariation.FULL_OUTER)) {
+ return null;
+ }
+ return Arrays.toString(vectorMapJoinInfo.getNonOuterSmallTableKeyColumnMap());
+ }
+
+ @Explain(vectorization = Vectorization.DETAIL, displayName = "outerSmallTableKeyMapping",
+ explainLevels = { Level.DEFAULT, Level.EXTENDED })
+ public List<String> getOuterSmallTableKeyMapping() {
+ if (!isNative ||
+ vectorMapJoinDesc.getVectorMapJoinVariation() != VectorMapJoinVariation.OUTER) {
+ return null;
+ }
+ return columnMappingToStringList(vectorMapJoinInfo.getOuterSmallTableKeyMapping());
+ }
+
+ @Explain(vectorization = Vectorization.DETAIL, displayName = "fullOuterSmallTableKeyMapping",
+ explainLevels = { Level.DEFAULT, Level.EXTENDED })
+ public List<String> getFullOuterSmallTableKeyMapping() {
+ if (!isNative ||
+ vectorMapJoinDesc.getVectorMapJoinVariation() != VectorMapJoinVariation.FULL_OUTER) {
return null;
}
- return outputColumnsToStringList(vectorMapJoinInfo.getBigTableRetainedMapping());
+ return columnMappingToStringList(vectorMapJoinInfo.getFullOuterSmallTableKeyMapping());
}
- @Explain(vectorization = Vectorization.OPERATOR, displayName = "nativeNotSupportedKeyTypes", explainLevels = { Level.DEFAULT, Level.EXTENDED })
+ @Explain(vectorization = Vectorization.OPERATOR, displayName = "nativeNotSupportedKeyTypes",
+ explainLevels = { Level.DEFAULT, Level.EXTENDED })
public List<String> getNativeNotSupportedKeyTypes() {
return vectorMapJoinDesc.getNotSupportedKeyTypes();
}
}
- @Explain(vectorization = Vectorization.OPERATOR, displayName = "Map Join Vectorization", explainLevels = { Level.DEFAULT, Level.EXTENDED })
+ @Explain(vectorization = Vectorization.OPERATOR, displayName = "Map Join Vectorization",
+ explainLevels = { Level.DEFAULT, Level.EXTENDED })
public MapJoinOperatorExplainVectorization getMapJoinVectorization() {
VectorMapJoinDesc vectorMapJoinDesc = (VectorMapJoinDesc) getVectorDesc();
if (vectorMapJoinDesc == null || this instanceof SMBJoinDesc) {
@@ -587,7 +738,8 @@ public class MapJoinDesc extends JoinDesc implements Serializable {
}
// Handle dual nature.
- @Explain(vectorization = Vectorization.OPERATOR, displayName = "SMB Map Join Vectorization", explainLevels = { Level.DEFAULT, Level.EXTENDED })
+ @Explain(vectorization = Vectorization.OPERATOR, displayName = "SMB Map Join Vectorization",
+ explainLevels = { Level.DEFAULT, Level.EXTENDED })
public SMBJoinOperatorExplainVectorization getSMBJoinVectorization() {
VectorSMBJoinDesc vectorSMBJoinDesc = (VectorSMBJoinDesc) getVectorDesc();
if (vectorSMBJoinDesc == null || !(this instanceof SMBJoinDesc)) {
http://git-wip-us.apache.org/repos/asf/hive/blob/a37827ec/ql/src/java/org/apache/hadoop/hive/ql/plan/MergeJoinWork.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/MergeJoinWork.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/MergeJoinWork.java
index 45f3347..dd907ef 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/MergeJoinWork.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/MergeJoinWork.java
@@ -29,7 +29,9 @@ import org.apache.hadoop.hive.ql.exec.CommonMergeJoinOperator;
import org.apache.hadoop.hive.ql.exec.HashTableDummyOperator;
import org.apache.hadoop.hive.ql.exec.Operator;
import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator;
+import org.apache.hadoop.hive.ql.plan.BaseWork.BaseExplainVectorization;
import org.apache.hadoop.hive.ql.plan.Explain.Level;
+import org.apache.hadoop.hive.ql.plan.Explain.Vectorization;
import org.apache.hadoop.mapred.JobConf;
public class MergeJoinWork extends BaseWork {
@@ -180,4 +182,59 @@ public class MergeJoinWork extends BaseWork {
public void addDummyOp(HashTableDummyOperator dummyOp) {
getMainWork().addDummyOp(dummyOp);
}
+
+ /**
+ * For now, this class just says in EXPLAIN VECTORIZATION we don't support vectorization of the
+ * Merge Join vertex instead of being silent about it.
+ */
+ public class MergeJoinExplainVectorization extends BaseExplainVectorization {
+
+ private final MergeJoinWork mergeJoinWork;
+
+ private VectorizationCondition[] mergeWorkVectorizationConditions;
+
+ public MergeJoinExplainVectorization(MergeJoinWork mergeJoinWork) {
+ super(mergeJoinWork);
+ this.mergeJoinWork = mergeJoinWork;
+ }
+
+ private VectorizationCondition[] createMergeWorkExplainVectorizationConditions() {
+
+ boolean enabled = false;
+
+ VectorizationCondition[] conditions = new VectorizationCondition[] {
+ new VectorizationCondition(
+ enabled,
+ "Vectorizing MergeJoin Supported")
+ };
+ return conditions;
+ }
+
+ @Explain(vectorization = Vectorization.SUMMARY, displayName = "enableConditionsMet",
+ explainLevels = { Level.DEFAULT, Level.EXTENDED })
+ public List<String> getEnableConditionsMet() {
+ if (mergeWorkVectorizationConditions == null) {
+ mergeWorkVectorizationConditions = createMergeWorkExplainVectorizationConditions();
+ }
+ return VectorizationCondition.getConditionsMet(mergeWorkVectorizationConditions);
+ }
+
+ @Explain(vectorization = Vectorization.SUMMARY, displayName = "enableConditionsNotMet",
+ explainLevels = { Level.DEFAULT, Level.EXTENDED })
+ public List<String> getEnableConditionsNotMet() {
+ if (mergeWorkVectorizationConditions == null) {
+ mergeWorkVectorizationConditions = createMergeWorkExplainVectorizationConditions();
+ }
+ return VectorizationCondition.getConditionsNotMet(mergeWorkVectorizationConditions);
+ }
+ }
+
+ @Explain(vectorization = Vectorization.SUMMARY, displayName = "MergeJoin Vectorization",
+ explainLevels = { Level.DEFAULT, Level.EXTENDED })
+ public MergeJoinExplainVectorization getReduceExplainVectorization() {
+ if (!getVectorizationExamined()) {
+ return null;
+ }
+ return new MergeJoinExplainVectorization(this);
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/a37827ec/ql/src/java/org/apache/hadoop/hive/ql/plan/OperatorExplainVectorization.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/OperatorExplainVectorization.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/OperatorExplainVectorization.java
index 446b810..5439e14 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/OperatorExplainVectorization.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/OperatorExplainVectorization.java
@@ -59,6 +59,25 @@ public class OperatorExplainVectorization {
return Arrays.toString(outputColumns);
}
+ public List<String> outputColumnsAndTypesToStringList(int[] outputColumns, TypeInfo[] typeInfos) {
+ final int size = outputColumns.length;
+ ArrayList<String> result = new ArrayList<String>(size);
+ for (int i = 0; i < size; i++) {
+ result.add(outputColumns[i] + ":" + typeInfos[i].toString());
+ }
+ return result;
+ }
+
+ public List<String> outputColumnsAndTypesToStringList(VectorColumnMapping vectorColumnMapping) {
+ final int size = vectorColumnMapping.getCount();
+ if (size == 0) {
+ return null;
+ }
+ int[] outputColumns = vectorColumnMapping.getOutputColumns();
+ TypeInfo[] typeInfos = vectorColumnMapping.getTypeInfos();
+ return outputColumnsAndTypesToStringList(outputColumns, typeInfos);
+ }
+
public List<String> columnMappingToStringList(VectorColumnMapping vectorColumnMapping) {
final int size = vectorColumnMapping.getCount();
if (size == 0) {
http://git-wip-us.apache.org/repos/asf/hive/blob/a37827ec/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceSinkDesc.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceSinkDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceSinkDesc.java
index 61216bc..b1d8e1f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceSinkDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceSinkDesc.java
@@ -29,6 +29,7 @@ import java.util.Set;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.io.AcidUtils;
import org.apache.hadoop.hive.ql.optimizer.signature.Signature;
+import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.plan.Explain.Level;
import org.apache.hadoop.hive.ql.plan.Explain.Vectorization;
import org.apache.hadoop.hive.ql.plan.VectorReduceSinkDesc.ReduceSinkKeyType;
@@ -193,6 +194,17 @@ public class ReduceSinkDesc extends AbstractOperatorDesc {
return outputKeyColumnNames;
}
+ // NOTE: Debugging only.
+ @Explain(displayName = "output key column names", explainLevels = { Level.DEBUG })
+ public List<String> getOutputKeyColumnNamesDisplay() {
+ List<String> result = new ArrayList<String>();
+ for (String name : outputKeyColumnNames) {
+ result.add(Utilities.ReduceField.KEY.name() + "." + name);
+ }
+ return result;
+ }
+
+
public void setOutputKeyColumnNames(
java.util.ArrayList<java.lang.String> outputKeyColumnNames) {
this.outputKeyColumnNames = outputKeyColumnNames;
@@ -202,6 +214,16 @@ public class ReduceSinkDesc extends AbstractOperatorDesc {
return outputValueColumnNames;
}
+ // NOTE: Debugging only.
+ @Explain(displayName = "output value column names", explainLevels = { Level.DEBUG })
+ public List<String> getOutputValueColumnNamesDisplay() {
+ List<String> result = new ArrayList<String>();
+ for (String name : outputValueColumnNames) {
+ result.add(Utilities.ReduceField.VALUE.name() + "." + name);
+ }
+ return result;
+ }
+
public void setOutputValueColumnNames(
java.util.ArrayList<java.lang.String> outputValueColumnNames) {
this.outputValueColumnNames = outputValueColumnNames;
@@ -540,34 +562,41 @@ public class ReduceSinkDesc extends AbstractOperatorDesc {
return vectorExpressionsToStringList(vectorReduceSinkInfo.getReduceSinkValueExpressions());
}
- @Explain(vectorization = Vectorization.DETAIL, displayName = "keyColumnNums", explainLevels = { Level.DEFAULT, Level.EXTENDED })
- public String getKeyColumnNums() {
+ @Explain(vectorization = Vectorization.DETAIL, displayName = "keyColumns",
+ explainLevels = { Level.DEFAULT, Level.EXTENDED })
+ public List<String> getKeyColumns() {
if (!isNative) {
return null;
}
int[] keyColumnMap = vectorReduceSinkInfo.getReduceSinkKeyColumnMap();
if (keyColumnMap == null) {
// Always show an array.
- keyColumnMap = new int[0];
+ return new ArrayList<String>();
}
- return Arrays.toString(keyColumnMap);
+ return outputColumnsAndTypesToStringList(
+ vectorReduceSinkInfo.getReduceSinkKeyColumnMap(),
+ vectorReduceSinkInfo.getReduceSinkKeyTypeInfos());
}
- @Explain(vectorization = Vectorization.DETAIL, displayName = "valueColumnNums", explainLevels = { Level.DEFAULT, Level.EXTENDED })
- public String getValueColumnNums() {
+ @Explain(vectorization = Vectorization.DETAIL, displayName = "valueColumns",
+ explainLevels = { Level.DEFAULT, Level.EXTENDED })
+ public List<String> getValueColumns() {
if (!isNative) {
return null;
}
int[] valueColumnMap = vectorReduceSinkInfo.getReduceSinkValueColumnMap();
if (valueColumnMap == null) {
// Always show an array.
- valueColumnMap = new int[0];
+ return new ArrayList<String>();
}
- return Arrays.toString(valueColumnMap);
+ return outputColumnsAndTypesToStringList(
+ vectorReduceSinkInfo.getReduceSinkValueColumnMap(),
+ vectorReduceSinkInfo.getReduceSinkValueTypeInfos());
}
- @Explain(vectorization = Vectorization.DETAIL, displayName = "bucketColumnNums", explainLevels = { Level.DEFAULT, Level.EXTENDED })
- public String getBucketColumnNums() {
+ @Explain(vectorization = Vectorization.DETAIL, displayName = "bucketColumns",
+ explainLevels = { Level.DEFAULT, Level.EXTENDED })
+ public List<String> getBucketColumns() {
if (!isNative) {
return null;
}
@@ -576,11 +605,14 @@ public class ReduceSinkDesc extends AbstractOperatorDesc {
// Suppress empty column map.
return null;
}
- return Arrays.toString(bucketColumnMap);
+ return outputColumnsAndTypesToStringList(
+ vectorReduceSinkInfo.getReduceSinkBucketColumnMap(),
+ vectorReduceSinkInfo.getReduceSinkBucketTypeInfos());
}
- @Explain(vectorization = Vectorization.DETAIL, displayName = "partitionColumnNums", explainLevels = { Level.DEFAULT, Level.EXTENDED })
- public String getPartitionColumnNums() {
+ @Explain(vectorization = Vectorization.DETAIL, displayName = "partitionColumns",
+ explainLevels = { Level.DEFAULT, Level.EXTENDED })
+ public List<String> getPartitionColumns() {
if (!isNative) {
return null;
}
@@ -589,7 +621,9 @@ public class ReduceSinkDesc extends AbstractOperatorDesc {
// Suppress empty column map.
return null;
}
- return Arrays.toString(partitionColumnMap);
+ return outputColumnsAndTypesToStringList(
+ vectorReduceSinkInfo.getReduceSinkPartitionColumnMap(),
+ vectorReduceSinkInfo.getReduceSinkPartitionTypeInfos());
}
private VectorizationCondition[] createNativeConditions() {
@@ -598,7 +632,8 @@ public class ReduceSinkDesc extends AbstractOperatorDesc {
String engine = vectorReduceSinkDesc.getEngine();
String engineInSupportedCondName =
- HiveConf.ConfVars.HIVE_EXECUTION_ENGINE.varname + " " + engine + " IN " + vectorizableReduceSinkNativeEngines;
+ HiveConf.ConfVars.HIVE_EXECUTION_ENGINE.varname + " " + engine + " IN " +
+ vectorizableReduceSinkNativeEngines;
boolean engineInSupported = vectorizableReduceSinkNativeEngines.contains(engine);
VectorizationCondition[] conditions = new VectorizationCondition[] {
@@ -633,7 +668,8 @@ public class ReduceSinkDesc extends AbstractOperatorDesc {
return conditions;
}
- @Explain(vectorization = Vectorization.OPERATOR, displayName = "nativeConditionsMet", explainLevels = { Level.DEFAULT, Level.EXTENDED })
+ @Explain(vectorization = Vectorization.OPERATOR, displayName = "nativeConditionsMet",
+ explainLevels = { Level.DEFAULT, Level.EXTENDED })
public List<String> getNativeConditionsMet() {
if (nativeConditions == null) {
nativeConditions = createNativeConditions();
@@ -641,7 +677,8 @@ public class ReduceSinkDesc extends AbstractOperatorDesc {
return VectorizationCondition.getConditionsMet(nativeConditions);
}
- @Explain(vectorization = Vectorization.OPERATOR, displayName = "nativeConditionsNotMet", explainLevels = { Level.DEFAULT, Level.EXTENDED })
+ @Explain(vectorization = Vectorization.OPERATOR, displayName = "nativeConditionsNotMet",
+ explainLevels = { Level.DEFAULT, Level.EXTENDED })
public List<String> getNativeConditionsNotMet() {
if (nativeConditions == null) {
nativeConditions = createNativeConditions();
@@ -650,7 +687,8 @@ public class ReduceSinkDesc extends AbstractOperatorDesc {
}
}
- @Explain(vectorization = Vectorization.OPERATOR, displayName = "Reduce Sink Vectorization", explainLevels = { Level.DEFAULT, Level.EXTENDED })
+ @Explain(vectorization = Vectorization.OPERATOR, displayName = "Reduce Sink Vectorization",
+ explainLevels = { Level.DEFAULT, Level.EXTENDED })
public ReduceSinkOperatorExplainVectorization getReduceSinkVectorization() {
VectorReduceSinkDesc vectorReduceSinkDesc = (VectorReduceSinkDesc) getVectorDesc();
if (vectorReduceSinkDesc == null) {
http://git-wip-us.apache.org/repos/asf/hive/blob/a37827ec/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorMapJoinDesc.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorMapJoinDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorMapJoinDesc.java
index 58032ca..89a07b4 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorMapJoinDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorMapJoinDesc.java
@@ -85,11 +85,11 @@ public class VectorMapJoinDesc extends AbstractVectorDesc {
}
public static enum VectorMapJoinVariation {
- NONE,
- INNER_BIG_ONLY,
INNER,
+ INNER_BIG_ONLY,
LEFT_SEMI,
- OUTER
+ OUTER,
+ FULL_OUTER
}
private HashTableImplementationType hashTableImplementationType;
@@ -107,7 +107,7 @@ public class VectorMapJoinDesc extends AbstractVectorDesc {
hashTableImplementationType = HashTableImplementationType.NONE;
hashTableKind = HashTableKind.NONE;
hashTableKeyType = HashTableKeyType.NONE;
- vectorMapJoinVariation = VectorMapJoinVariation.NONE;
+ vectorMapJoinVariation = null;
minMaxEnabled = false;
allBigTableKeyExpressions = null;
@@ -206,6 +206,7 @@ public class VectorMapJoinDesc extends AbstractVectorDesc {
private List<String> notSupportedKeyTypes;
private boolean smallTableExprVectorizes;
private boolean outerJoinHasNoKeys;
+ boolean isFullOuter;
public void setUseOptimizedTable(boolean useOptimizedTable) {
this.useOptimizedTable = useOptimizedTable;
@@ -274,5 +275,10 @@ public class VectorMapJoinDesc extends AbstractVectorDesc {
public boolean getIsHybridHashJoin() {
return isHybridHashJoin;
}
-
+ public void setIsFullOuter(boolean isFullOuter) {
+ this.isFullOuter = isFullOuter;
+ }
+ public boolean getIsFullOuter() {
+ return isFullOuter;
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/a37827ec/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorMapJoinInfo.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorMapJoinInfo.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorMapJoinInfo.java
index 6db0540..ad82e5c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorMapJoinInfo.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorMapJoinInfo.java
@@ -48,9 +48,19 @@ public class VectorMapJoinInfo {
private TypeInfo[] bigTableValueTypeInfos;
private VectorExpression[] slimmedBigTableValueExpressions;
- private VectorColumnOutputMapping bigTableRetainedMapping;
- private VectorColumnOutputMapping bigTableOuterKeyMapping;
- private VectorColumnSourceMapping smallTableMapping;
+ private VectorExpression[] bigTableFilterExpressions;
+
+ private int[] bigTableRetainColumnMap;
+ private TypeInfo[] bigTableRetainTypeInfos;
+
+ private int[] nonOuterSmallTableKeyColumnMap;
+ private TypeInfo[] nonOuterSmallTableKeyTypeInfos;
+
+ private VectorColumnOutputMapping outerSmallTableKeyMapping;
+
+ private VectorColumnSourceMapping fullOuterSmallTableKeyMapping;
+
+ private VectorColumnSourceMapping smallTableValueMapping;
private VectorColumnSourceMapping projectionMapping;
@@ -65,9 +75,19 @@ public class VectorMapJoinInfo {
bigTableValueTypeInfos = null;
slimmedBigTableValueExpressions = null;
- bigTableRetainedMapping = null;
- bigTableOuterKeyMapping = null;
- smallTableMapping = null;
+ bigTableFilterExpressions = null;
+
+ bigTableRetainColumnMap = null;
+ bigTableRetainTypeInfos = null;
+
+ nonOuterSmallTableKeyColumnMap = null;
+ nonOuterSmallTableKeyTypeInfos = null;
+
+ outerSmallTableKeyMapping = null;
+
+ fullOuterSmallTableKeyMapping = null;
+
+ smallTableValueMapping = null;
projectionMapping = null;
}
@@ -138,28 +158,69 @@ public class VectorMapJoinInfo {
this.slimmedBigTableValueExpressions = slimmedBigTableValueExpressions;
}
- public void setBigTableRetainedMapping(VectorColumnOutputMapping bigTableRetainedMapping) {
- this.bigTableRetainedMapping = bigTableRetainedMapping;
+ public VectorExpression[] getBigTableFilterExpressions() {
+ return bigTableFilterExpressions;
+ }
+
+ public void setBigTableFilterExpressions(VectorExpression[] bigTableFilterExpressions) {
+ this.bigTableFilterExpressions = bigTableFilterExpressions;
+ }
+
+ public void setBigTableRetainColumnMap(int[] bigTableRetainColumnMap) {
+ this.bigTableRetainColumnMap = bigTableRetainColumnMap;
+ }
+
+ public int[] getBigTableRetainColumnMap() {
+ return bigTableRetainColumnMap;
+ }
+
+ public void setBigTableRetainTypeInfos(TypeInfo[] bigTableRetainTypeInfos) {
+ this.bigTableRetainTypeInfos = bigTableRetainTypeInfos;
+ }
+
+ public TypeInfo[] getBigTableRetainTypeInfos() {
+ return bigTableRetainTypeInfos;
+ }
+
+ public void setNonOuterSmallTableKeyColumnMap(int[] nonOuterSmallTableKeyColumnMap) {
+ this.nonOuterSmallTableKeyColumnMap = nonOuterSmallTableKeyColumnMap;
+ }
+
+ public int[] getNonOuterSmallTableKeyColumnMap() {
+ return nonOuterSmallTableKeyColumnMap;
+ }
+
+ public void setNonOuterSmallTableKeyTypeInfos(TypeInfo[] nonOuterSmallTableKeyTypeInfos) {
+ this.nonOuterSmallTableKeyTypeInfos = nonOuterSmallTableKeyTypeInfos;
+ }
+
+ public TypeInfo[] getNonOuterSmallTableKeyTypeInfos() {
+ return nonOuterSmallTableKeyTypeInfos;
+ }
+
+ public void setOuterSmallTableKeyMapping(VectorColumnOutputMapping outerSmallTableKeyMapping) {
+ this.outerSmallTableKeyMapping = outerSmallTableKeyMapping;
}
- public VectorColumnOutputMapping getBigTableRetainedMapping() {
- return bigTableRetainedMapping;
+ public VectorColumnOutputMapping getOuterSmallTableKeyMapping() {
+ return outerSmallTableKeyMapping;
}
- public void setBigTableOuterKeyMapping(VectorColumnOutputMapping bigTableOuterKeyMapping) {
- this.bigTableOuterKeyMapping = bigTableOuterKeyMapping;
+ public void setFullOuterSmallTableKeyMapping(
+ VectorColumnSourceMapping fullOuterSmallTableKeyMapping) {
+ this.fullOuterSmallTableKeyMapping = fullOuterSmallTableKeyMapping;
}
- public VectorColumnOutputMapping getBigTableOuterKeyMapping() {
- return bigTableOuterKeyMapping;
+ public VectorColumnSourceMapping getFullOuterSmallTableKeyMapping() {
+ return fullOuterSmallTableKeyMapping;
}
- public void setSmallTableMapping(VectorColumnSourceMapping smallTableMapping) {
- this.smallTableMapping = smallTableMapping;
+ public void setSmallTableValueMapping(VectorColumnSourceMapping smallTableValueMapping) {
+ this.smallTableValueMapping = smallTableValueMapping;
}
- public VectorColumnSourceMapping getSmallTableMapping() {
- return smallTableMapping;
+ public VectorColumnSourceMapping getSmallTableValueMapping() {
+ return smallTableValueMapping;
}
public void setProjectionMapping(VectorColumnSourceMapping projectionMapping) {
http://git-wip-us.apache.org/repos/asf/hive/blob/a37827ec/ql/src/test/org/apache/hadoop/hive/ql/exec/persistence/TestBytesBytesMultiHashMap.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/persistence/TestBytesBytesMultiHashMap.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/persistence/TestBytesBytesMultiHashMap.java
index 9f785e6..e5c749f 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/exec/persistence/TestBytesBytesMultiHashMap.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/persistence/TestBytesBytesMultiHashMap.java
@@ -84,9 +84,9 @@ public class TestBytesBytesMultiHashMap {
map.put(kv2, -1);
key[0] = (byte)(key[0] + 1);
BytesBytesMultiHashMap.Result hashMapResult = new BytesBytesMultiHashMap.Result();
- map.getValueResult(key, 0, key.length, hashMapResult);
+ map.getValueResult(key, 0, key.length, hashMapResult, null);
assertTrue(!hashMapResult.hasRows());
- map.getValueResult(key, 0, 0, hashMapResult);
+ map.getValueResult(key, 0, 0, hashMapResult, null);
assertTrue(!hashMapResult.hasRows());
}
@@ -104,7 +104,7 @@ public class TestBytesBytesMultiHashMap {
assertEquals(CAPACITY, map.getCapacity());
// Get of non-existent key should terminate..
BytesBytesMultiHashMap.Result hashMapResult = new BytesBytesMultiHashMap.Result();
- map.getValueResult(new byte[0], 0, 0, hashMapResult);
+ map.getValueResult(new byte[0], 0, 0, hashMapResult, null);
}
@Test
@@ -123,7 +123,7 @@ public class TestBytesBytesMultiHashMap {
private void verifyHashMapResult(BytesBytesMultiHashMap map, byte[] key, byte[]... values) {
BytesBytesMultiHashMap.Result hashMapResult = new BytesBytesMultiHashMap.Result();
- byte state = map.getValueResult(key, 0, key.length, hashMapResult);
+ byte state = map.getValueResult(key, 0, key.length, hashMapResult, null);
HashSet<ByteBuffer> hs = new HashSet<ByteBuffer>();
int count = 0;
if (hashMapResult.hasRows()) {
http://git-wip-us.apache.org/repos/asf/hive/blob/a37827ec/ql/src/test/org/apache/hadoop/hive/ql/exec/util/collectoroperator/CollectorTestOperator.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/util/collectoroperator/CollectorTestOperator.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/util/collectoroperator/CollectorTestOperator.java
index 6491d79..244208b 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/exec/util/collectoroperator/CollectorTestOperator.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/util/collectoroperator/CollectorTestOperator.java
@@ -26,8 +26,22 @@ public class CollectorTestOperator extends Operator {
private static final long serialVersionUID = 1L;
+ private boolean isClosed;
+ private boolean isAborted;
+
public CollectorTestOperator() {
super();
+
+ isClosed = false;
+ isAborted = false;
+ }
+
+ public boolean getIsClosed() {
+ return isClosed;
+ }
+
+ public boolean getIsAborted() {
+ return isAborted;
}
@Override
@@ -36,6 +50,14 @@ public class CollectorTestOperator extends Operator {
}
@Override
+ public void closeOp(boolean abort) {
+ isClosed = true;
+ if (abort) {
+ isAborted = true;
+ }
+ }
+
+ @Override
public String getName() {
return CollectorTestOperator.class.getSimpleName();
}