You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by mm...@apache.org on 2018/09/16 14:05:46 UTC

[42/47] hive git commit: HIVE-18908: FULL OUTER JOIN to MapJoin (Matt McCline, reviewed by Teddy Choi)

http://git-wip-us.apache.org/repos/asf/hive/blob/a37827ec/ql/src/java/org/apache/hadoop/hive/ql/optimizer/FullOuterMapJoinOptimization.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/FullOuterMapJoinOptimization.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/FullOuterMapJoinOptimization.java
new file mode 100644
index 0000000..b9e86eb
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/FullOuterMapJoinOptimization.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.optimizer;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Properties;
+import org.apache.hadoop.hive.ql.parse.SemanticException;
+import org.apache.hadoop.hive.ql.plan.MapJoinDesc;
+import org.apache.hadoop.hive.ql.plan.TableDesc;
+import org.apache.hadoop.hive.serde.serdeConstants;
+import org.apache.hadoop.hive.serde2.SerDeUtils;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
+
+import com.google.common.base.Joiner;
+
+/**
+ * FULL OUTER MapJoin planning.
+ */
+public class FullOuterMapJoinOptimization {
+
+  FullOuterMapJoinOptimization() {
+  }
+
+  public static void removeFilterMap(MapJoinDesc mapJoinDesc) throws SemanticException {
+    int[][] filterMaps = mapJoinDesc.getFilterMap();
+    if (filterMaps == null) {
+      return;
+    }
+    final byte posBigTable = (byte) mapJoinDesc.getPosBigTable();
+    final int numAliases = mapJoinDesc.getExprs().size();
+    List<TableDesc> valueFilteredTblDescs = mapJoinDesc.getValueFilteredTblDescs();
+    for (byte pos = 0; pos < numAliases; pos++) {
+      if (pos != posBigTable) {
+        int[] filterMap = filterMaps[pos];
+        TableDesc tableDesc = valueFilteredTblDescs.get(pos);
+        Properties properties = tableDesc.getProperties();
+        String columnNameProperty = properties.getProperty(serdeConstants.LIST_COLUMNS);
+        String columnNameDelimiter =
+            properties.containsKey(serdeConstants.COLUMN_NAME_DELIMITER) ?
+                properties.getProperty(serdeConstants.COLUMN_NAME_DELIMITER) :
+                  String.valueOf(SerDeUtils.COMMA);
+
+        String columnTypeProperty = properties.getProperty(serdeConstants.LIST_COLUMN_TYPES);
+        List<String> columnNameList;
+        if (columnNameProperty.length() == 0) {
+          columnNameList = new ArrayList<String>();
+        } else {
+          columnNameList = Arrays.asList(columnNameProperty.split(columnNameDelimiter));
+        }
+        List<String> truncatedColumnNameList = columnNameList.subList(0, columnNameList.size() - 1);
+        String truncatedColumnNameProperty =
+            Joiner.on(columnNameDelimiter).join(truncatedColumnNameList);
+
+        List<TypeInfo> columnTypeList;
+        if (columnTypeProperty.length() == 0) {
+          columnTypeList = new ArrayList<TypeInfo>();
+        } else {
+          columnTypeList = TypeInfoUtils
+              .getTypeInfosFromTypeString(columnTypeProperty);
+        }
+        if (!columnTypeList.get(columnTypeList.size() - 1).equals(TypeInfoFactory.shortTypeInfo)) {
+          throw new SemanticException("Expecting filterTag smallint as last column type");
+        }
+        List<TypeInfo> truncatedColumnTypeList =
+            columnTypeList.subList(0, columnTypeList.size() - 1);
+        String truncatedColumnTypeProperty =
+            Joiner.on(",").join(truncatedColumnTypeList);
+
+        properties.setProperty(serdeConstants.LIST_COLUMNS, truncatedColumnNameProperty);
+        properties.setProperty(serdeConstants.LIST_COLUMN_TYPES, truncatedColumnTypeProperty);
+      }
+    }
+    mapJoinDesc.setFilterMap(null);
+  }
+}

http://git-wip-us.apache.org/repos/asf/hive/blob/a37827ec/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java
index 019372b..23186c6 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java
@@ -26,6 +26,7 @@ import java.util.Iterator;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Map.Entry;
 import java.util.Set;
 import java.util.Stack;
 
@@ -48,6 +49,7 @@ import org.apache.hadoop.hive.ql.exec.SMBMapJoinOperator;
 import org.apache.hadoop.hive.ql.exec.ScriptOperator;
 import org.apache.hadoop.hive.ql.exec.SelectOperator;
 import org.apache.hadoop.hive.ql.exec.UnionOperator;
+import org.apache.hadoop.hive.ql.exec.persistence.MapJoinKey;
 import org.apache.hadoop.hive.ql.lib.DefaultRuleDispatcher;
 import org.apache.hadoop.hive.ql.lib.Dispatcher;
 import org.apache.hadoop.hive.ql.lib.GraphWalker;
@@ -56,6 +58,7 @@ import org.apache.hadoop.hive.ql.lib.NodeProcessor;
 import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx;
 import org.apache.hadoop.hive.ql.lib.Rule;
 import org.apache.hadoop.hive.ql.lib.RuleRegExp;
+import org.apache.hadoop.hive.ql.optimizer.physical.Vectorizer.EnabledOverride;
 import org.apache.hadoop.hive.ql.parse.GenMapRedWalker;
 import org.apache.hadoop.hive.ql.parse.ParseContext;
 import org.apache.hadoop.hive.ql.parse.SemanticException;
@@ -75,6 +78,7 @@ import org.apache.hadoop.hive.ql.plan.SMBJoinDesc;
 import org.apache.hadoop.hive.ql.plan.SelectDesc;
 import org.apache.hadoop.hive.ql.plan.TableDesc;
 import org.apache.hadoop.hive.serde.serdeConstants;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
 import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -336,6 +340,9 @@ public class MapJoinProcessor extends Transform {
     // create the map-join operator
     MapJoinOperator mapJoinOp = convertJoinOpMapJoinOp(conf,
         op, leftInputJoin, baseSrc, mapAliases, mapJoinPos, noCheckOuterJoin);
+    if (mapJoinOp == null) {
+      return null;
+    }
 
     // remove old parents
     for (pos = 0; pos < newParentOps.size(); pos++) {
@@ -356,6 +363,213 @@ public class MapJoinProcessor extends Transform {
     return mapJoinOp;
   }
 
+  public static boolean onExpressionHasNullSafes(JoinDesc desc) {
+    boolean[] nullSafes = desc.getNullSafes();
+    if (nullSafes == null) {
+      return false;
+    }
+    for (boolean nullSafe : nullSafes) {
+      if (nullSafe) {
+        return true;
+      }
+    }
+    return false;
+  }
+
+  private static boolean checkFullOuterMapJoinCompatible(HiveConf hiveConf,
+      JoinOperator joinOp) throws SemanticException {
+    JoinDesc joinDesc = joinOp.getConf();
+
+    // Make sure all key and value expressions are columns.
+    for (Entry<Byte, List<ExprNodeDesc>> mapEntry : joinDesc.getExprs().entrySet()) {
+      List<ExprNodeDesc> exprList = mapEntry.getValue();
+      for (ExprNodeDesc expr : exprList) {
+        if (!(expr instanceof ExprNodeColumnDesc)) {
+          LOG.debug("FULL OUTER MapJoin: only column expressions are supported " + expr.toString());
+          return false;
+        }
+      }
+    }
+
+    // Check for supported key data types.
+    Byte[] order = joinDesc.getTagOrder();
+    ExprNodeDesc[][] joinKeysArray = joinDesc.getJoinKeys();
+    for (int i = 0; i < order.length; i++) {
+      byte pos = order[i];
+      ExprNodeDesc[] keyExprs = joinKeysArray[pos];
+      for (ExprNodeDesc keyExpr : keyExprs) {
+        TypeInfo typeInfo = keyExpr.getTypeInfo();
+
+        // Verify we handle the key column types for an optimized table.  This is the effectively
+        // the same check used in Tez HashTableLoader.
+        if (!MapJoinKey.isSupportedField(typeInfo)) {
+          if (LOG.isDebugEnabled()) {
+            LOG.debug("FULL OUTER MapJoin not enabled: " +
+                " key type " + typeInfo.toString() + " not supported");
+          }
+          return false;
+        }
+      }
+    }
+
+    if (onExpressionHasNullSafes(joinDesc)) {
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("FULL OUTER MapJoin not enabled: " +
+            "nullsafe not supported");
+      }
+      return false;
+    }
+
+    boolean isVectorizationMapJoinNativeEnabled = HiveConf.getBoolVar(hiveConf,
+        HiveConf.ConfVars.HIVE_VECTORIZATION_MAPJOIN_NATIVE_ENABLED);
+    boolean isHybridHashJoin = HiveConf.getBoolVar(hiveConf,
+        HiveConf.ConfVars.HIVEUSEHYBRIDGRACEHASHJOIN);
+    if (isVectorizationMapJoinNativeEnabled && isHybridHashJoin) {
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("FULL OUTER MapJoin not enabled: " +
+            "Native Vector MapJoin and Hybrid Grace not supported");
+      }
+      return false;
+    }
+
+    if (joinDesc.getResidualFilterExprs() != null &&
+        joinDesc.getResidualFilterExprs().size() != 0) {
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("FULL OUTER MapJoin not enabled: " +
+            "non-equi joins not supported");
+      }
+      return false;
+    }
+
+    return true;
+  }
+
+  public static boolean precheckFullOuter(HiveConf hiveConf, JoinOperator joinOp)
+      throws SemanticException {
+    JoinDesc joinDesc = joinOp.getConf();
+    JoinCondDesc[] conds = joinDesc.getConds();
+
+    /*
+     * Are we even being asked to do a FULL OUTER JOIN?
+     */
+    boolean hasFullOuterJoin = false;
+    for (JoinCondDesc cond : conds) {
+      if (cond.getType() == JoinDesc.FULL_OUTER_JOIN) {
+        hasFullOuterJoin = true;
+        break;
+      }
+    }
+    if (!hasFullOuterJoin) {
+      return false;
+    }
+
+    if (conds.length > 1) {
+
+      // No multiple condition FULL OUTER MapJoin.
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("FULL OUTER MapJoin not enabled: multiple JOIN conditions not supported");
+      }
+      return false;
+    }
+
+    return true;
+  }
+
+  public static boolean isFullOuterMapEnabled(HiveConf hiveConf, JoinOperator joinOp)
+      throws SemanticException {
+
+    final String testMapJoinFullOuterOverrideString =
+        HiveConf.getVar(hiveConf,
+            HiveConf.ConfVars.HIVE_TEST_MAPJOINFULLOUTER_OVERRIDE);
+    EnabledOverride mapJoinFullOuterOverride =
+        EnabledOverride.nameMap.get(testMapJoinFullOuterOverrideString);
+
+    final boolean isEnabled =
+        HiveConf.getBoolVar(
+            hiveConf,
+            HiveConf.ConfVars.HIVEMAPJOINFULLOUTER);
+    switch (mapJoinFullOuterOverride) {
+    case NONE:
+      {
+        if (!isEnabled) {
+          if (LOG.isDebugEnabled()) {
+            LOG.debug("FULL OUTER MapJoin not enabled: " +
+                HiveConf.ConfVars.HIVEMAPJOINFULLOUTER.varname + " is false");
+          }
+          return false;
+        }
+      }
+      break;
+    case DISABLE:
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("FULL OUTER MapJoin not enabled: " +
+            HiveConf.ConfVars.HIVE_TEST_MAPJOINFULLOUTER_OVERRIDE.varname + " is disable (" +
+            " " + HiveConf.ConfVars.HIVEMAPJOINFULLOUTER.varname + " is " + isEnabled + ")");
+      }
+      return false;
+    case ENABLE:
+
+      // Different parts of the code may rely on this being set...
+      HiveConf.setBoolVar(hiveConf,
+          HiveConf.ConfVars.HIVEMAPJOINFULLOUTER, true);
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("FULL OUTER MapJoin is enabled: " +
+            HiveConf.ConfVars.HIVE_TEST_MAPJOINFULLOUTER_OVERRIDE.varname + " is enable (" +
+            " " + HiveConf.ConfVars.HIVEMAPJOINFULLOUTER.varname + " is " + isEnabled + ")");
+      }
+      break;
+    default:
+      throw new RuntimeException("Unexpected vectorization enabled override " +
+          mapJoinFullOuterOverride);
+    }
+
+    final String engine =
+        HiveConf.getVar(
+            hiveConf,
+            HiveConf.ConfVars.HIVE_EXECUTION_ENGINE);
+    final boolean isTezEngine = engine.equalsIgnoreCase("tez");
+    if (!isTezEngine) {
+
+      // Only Tez for now.
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("FULL OUTER MapJoin not enabled: Only Tez engine supported");
+      }
+      return false;
+    }
+
+    /*
+     * Optimized Hash Table (i.e. not old-style MR HashMap).
+     */
+    final boolean isOptimizedHashTableEnabled =
+        HiveConf.getBoolVar(
+            hiveConf,
+            HiveConf.ConfVars.HIVEMAPJOINUSEOPTIMIZEDTABLE);
+    if (!isOptimizedHashTableEnabled) {
+      if (LOG.isDebugEnabled()) {
+        LOG.debug("FULL OUTER MapJoin not enabled: " +
+            HiveConf.ConfVars.HIVEMAPJOINUSEOPTIMIZEDTABLE.varname + " is false");
+      }
+      return false;
+    }
+
+    boolean isCompatibleFullOuterMapJoin = checkFullOuterMapJoinCompatible(hiveConf, joinOp);
+    if (!isCompatibleFullOuterMapJoin) {
+      return false;
+    }
+
+    if (LOG.isDebugEnabled()) {
+      LOG.debug("FULL OUTER MapJoin enabled");
+    }
+    return true;
+  }
+
+  public static boolean isFullOuterEnabledForDynamicPartitionHashJoin(HiveConf hiveConf, JoinOperator joinOp)
+      throws SemanticException {
+    JoinDesc joinDesc = joinOp.getConf();
+
+    return true;
+  }
+
   public static MapJoinOperator convertJoinOpMapJoinOp(HiveConf hconf,
       JoinOperator op, boolean leftInputJoin, String[] baseSrc, List<String> mapAliases,
       int mapJoinPos, boolean noCheckOuterJoin) throws SemanticException {
@@ -371,6 +585,9 @@ public class MapJoinProcessor extends Transform {
     MapJoinDesc mapJoinDescriptor =
         getMapJoinDesc(hconf, op, leftInputJoin, baseSrc, mapAliases,
             mapJoinPos, noCheckOuterJoin, adjustParentsChildren);
+    if (mapJoinDescriptor == null) {
+      return null;
+    }
     // reduce sink row resolver used to generate map join op
     RowSchema outputRS = op.getSchema();
 
@@ -474,6 +691,9 @@ public class MapJoinProcessor extends Transform {
     MapJoinOperator mapJoinOp = convertMapJoin(pctx.getConf(), op,
         op.getConf().isLeftInputJoin(), op.getConf().getBaseSrc(),
         op.getConf().getMapAliases(), mapJoinPos, noCheckOuterJoin, true);
+    if (mapJoinOp == null) {
+      return null;
+    }
 
     // create a dummy select to select all columns
     genSelectPlan(pctx, mapJoinOp);
@@ -502,8 +722,29 @@ public class MapJoinProcessor extends Transform {
    * @return set of big table candidates
    */
   public static Set<Integer> getBigTableCandidates(JoinCondDesc[] condns) {
+    return getBigTableCandidates(condns, /* isSupportFullOuter */ false);
+  }
+
+  public static Set<Integer> getBigTableCandidates(JoinCondDesc[] condns,
+      boolean isSupportFullOuter) {
+
     Set<Integer> bigTableCandidates = new HashSet<Integer>();
 
+    if (condns.length == 1) {
+      JoinCondDesc condn = condns[0];
+      if (condn.getType() == JoinDesc.FULL_OUTER_JOIN) {
+
+        if (!isSupportFullOuter) {
+          return new HashSet<Integer>();
+        }
+
+        // FULL OUTER MapJoin must be a single condition.
+        bigTableCandidates.add(condn.getLeft());
+        bigTableCandidates.add(condn.getRight());
+        return bigTableCandidates;
+      }
+    }
+
     boolean seenOuterJoin = false;
     Set<Integer> seenPostitions = new HashSet<Integer>();
     Set<Integer> leftPosListOfLastRightOuterJoin = new HashSet<Integer>();
@@ -512,17 +753,14 @@ public class MapJoinProcessor extends Transform {
     boolean lastSeenRightOuterJoin = false;
     for (JoinCondDesc condn : condns) {
       int joinType = condn.getType();
+      if (joinType == JoinDesc.FULL_OUTER_JOIN) {
+        return new HashSet<Integer>();
+      }
+
       seenPostitions.add(condn.getLeft());
       seenPostitions.add(condn.getRight());
 
-      if (joinType == JoinDesc.FULL_OUTER_JOIN) {
-        // setting these 2 parameters here just in case that if the code got
-        // changed in future, these 2 are not missing.
-        seenOuterJoin = true;
-        lastSeenRightOuterJoin = false;
-        // empty set - cannot convert
-        return new HashSet<Integer>();
-      } else if (joinType == JoinDesc.LEFT_OUTER_JOIN
+      if (joinType == JoinDesc.LEFT_OUTER_JOIN
           || joinType == JoinDesc.LEFT_SEMI_JOIN) {
         seenOuterJoin = true;
         if(bigTableCandidates.size() == 0) {
@@ -564,7 +802,8 @@ public class MapJoinProcessor extends Transform {
    * @throws SemanticException if given position is not in the big table candidates.
    */
   public static int checkMapJoin(int mapJoinPos, JoinCondDesc[] condns) {
-    Set<Integer> bigTableCandidates = MapJoinProcessor.getBigTableCandidates(condns);
+    Set<Integer> bigTableCandidates =
+        MapJoinProcessor.getBigTableCandidates(condns, /* isSupportFullOuter */ true);
 
     // bigTableCandidates can never be null
     if (!bigTableCandidates.contains(mapJoinPos)) {
@@ -1183,6 +1422,9 @@ public class MapJoinProcessor extends Transform {
     }
 
     List<ExprNodeDesc> keyCols = keyExprMap.get((byte) mapJoinPos);
+    if (keyCols == null) {
+      return null;
+    }
 
     List<String> outputColumnNames = op.getConf().getOutputColumnNames();
     TableDesc keyTableDesc =

http://git-wip-us.apache.org/repos/asf/hive/blob/a37827ec/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SparkMapJoinProcessor.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SparkMapJoinProcessor.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SparkMapJoinProcessor.java
index 16d088a..a5400d6 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SparkMapJoinProcessor.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SparkMapJoinProcessor.java
@@ -64,6 +64,9 @@ public class SparkMapJoinProcessor extends MapJoinProcessor {
     MapJoinOperator mapJoinOp = convertJoinOpMapJoinOp(conf,
         op, op.getConf().isLeftInputJoin(), op.getConf().getBaseSrc(),
         op.getConf().getMapAliases(), bigTablePos, noCheckOuterJoin);
+    if (mapJoinOp == null) {
+      return null;
+    }
 
     // 1. remove RS as parent for the big table branch
     // 2. remove old join op from child set of all the RSs

http://git-wip-us.apache.org/repos/asf/hive/blob/a37827ec/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
index 48974f8..51b186c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
@@ -73,6 +73,9 @@ import org.apache.hadoop.hive.ql.exec.vector.mapjoin.VectorMapJoinLeftSemiString
 import org.apache.hadoop.hive.ql.exec.vector.mapjoin.VectorMapJoinOuterLongOperator;
 import org.apache.hadoop.hive.ql.exec.vector.mapjoin.VectorMapJoinOuterMultiKeyOperator;
 import org.apache.hadoop.hive.ql.exec.vector.mapjoin.VectorMapJoinOuterStringOperator;
+import org.apache.hadoop.hive.ql.exec.vector.mapjoin.VectorMapJoinFullOuterLongOperator;
+import org.apache.hadoop.hive.ql.exec.vector.mapjoin.VectorMapJoinFullOuterMultiKeyOperator;
+import org.apache.hadoop.hive.ql.exec.vector.mapjoin.VectorMapJoinFullOuterStringOperator;
 import org.apache.hadoop.hive.ql.exec.vector.ptf.VectorPTFOperator;
 import org.apache.hadoop.hive.ql.exec.vector.udf.VectorUDFAdaptor;
 import org.apache.hadoop.hive.ql.exec.vector.ColumnVector;
@@ -122,6 +125,7 @@ import org.apache.hadoop.hive.ql.plan.LimitDesc;
 import org.apache.hadoop.hive.ql.plan.MapJoinDesc;
 import org.apache.hadoop.hive.ql.plan.MapWork;
 import org.apache.hadoop.hive.ql.plan.MapredWork;
+import org.apache.hadoop.hive.ql.plan.MergeJoinWork;
 import org.apache.hadoop.hive.ql.plan.OperatorDesc;
 import org.apache.hadoop.hive.ql.plan.PTFDesc;
 import org.apache.hadoop.hive.ql.plan.SelectDesc;
@@ -289,15 +293,15 @@ public class Vectorizer implements PhysicalPlanResolver {
 
   private HiveConf hiveConf;
 
-  public static enum VectorizationEnabledOverride {
+  public enum EnabledOverride {
     NONE,
     DISABLE,
     ENABLE;
 
-    public final static Map<String,VectorizationEnabledOverride> nameMap =
-        new HashMap<String,VectorizationEnabledOverride>();
+    public static final Map<String, EnabledOverride> nameMap =
+        new HashMap<String, EnabledOverride>();
     static {
-      for (VectorizationEnabledOverride vectorizationEnabledOverride : values()) {
+      for (EnabledOverride vectorizationEnabledOverride : values()) {
         nameMap.put(
             vectorizationEnabledOverride.name().toLowerCase(), vectorizationEnabledOverride);
       }
@@ -305,7 +309,7 @@ public class Vectorizer implements PhysicalPlanResolver {
   }
 
   boolean isVectorizationEnabled;
-  private VectorizationEnabledOverride vectorizationEnabledOverride;
+  private EnabledOverride vectorizationEnabledOverride;
   boolean isTestForcedVectorizationEnable;
 
   private boolean useVectorizedInputFileFormat;
@@ -725,10 +729,63 @@ public class Vectorizer implements PhysicalPlanResolver {
     }
   }
 
-  private List<Operator<? extends OperatorDesc>> newOperatorList() {
+  private static List<Operator<? extends OperatorDesc>> newOperatorList() {
     return new ArrayList<Operator<? extends OperatorDesc>>();
   }
 
+  public static void debugDisplayJoinOperatorTree(Operator<? extends OperatorDesc> joinOperator,
+      String prefix) {
+    List<Operator<? extends OperatorDesc>> currentParentList = newOperatorList();
+    currentParentList.add(joinOperator);
+
+    int depth = 0;
+    do {
+      List<Operator<? extends OperatorDesc>> nextParentList = newOperatorList();
+
+      final int count = currentParentList.size();
+      for (int i = 0; i < count; i++) {
+        Operator<? extends OperatorDesc> parent = currentParentList.get(i);
+        System.out.println(prefix + " parent depth " + depth + " " +
+            parent.getClass().getSimpleName() + " " + parent.toString());
+
+        List<Operator<? extends OperatorDesc>> parentList = parent.getParentOperators();
+        if (parentList == null || parentList.size() == 0) {
+          continue;
+        }
+
+        nextParentList.addAll(parentList);
+      }
+
+      currentParentList = nextParentList;
+      depth--;
+    } while (currentParentList.size() > 0);
+
+    List<Operator<? extends OperatorDesc>> currentChildList = newOperatorList();
+    currentChildList.addAll(joinOperator.getChildOperators());
+
+    depth = 1;
+    do {
+      List<Operator<? extends OperatorDesc>> nextChildList = newOperatorList();
+
+      final int count = currentChildList.size();
+      for (int i = 0; i < count; i++) {
+        Operator<? extends OperatorDesc> child = currentChildList.get(i);
+        System.out.println(prefix + " child depth " + depth + " " +
+            child.getClass().getSimpleName() + " " + child.toString());
+
+        List<Operator<? extends OperatorDesc>> childList = child.getChildOperators();
+        if (childList == null || childList.size() == 0) {
+          continue;
+        }
+
+        nextChildList.addAll(childList);
+      }
+
+      currentChildList = nextChildList;
+      depth--;
+    } while (currentChildList.size() > 0);
+  }
+
   private Operator<? extends OperatorDesc> validateAndVectorizeOperatorTree(
       Operator<? extends OperatorDesc> nonVecRootOperator,
       boolean isReduce, boolean isTezOrSpark,
@@ -973,7 +1030,15 @@ public class Vectorizer implements PhysicalPlanResolver {
             if (isReduceVectorizationEnabled) {
               convertReduceWork(reduceWork);
             }
+
             logReduceWorkExplainVectorization(reduceWork);
+          } else if (baseWork instanceof MergeJoinWork){
+            MergeJoinWork mergeJoinWork = (MergeJoinWork) baseWork;
+
+            // Always set the EXPLAIN conditions.
+            setMergeJoinWorkExplainConditions(mergeJoinWork);
+
+            logMergeJoinWorkExplainVectorization(mergeJoinWork);
           }
         }
       } else if (currTask instanceof SparkTask) {
@@ -1029,6 +1094,11 @@ public class Vectorizer implements PhysicalPlanResolver {
           HiveConf.getVar(hiveConf, HiveConf.ConfVars.HIVE_EXECUTION_ENGINE));
     }
 
+    private void setMergeJoinWorkExplainConditions(MergeJoinWork mergeJoinWork) {
+
+      setExplainConditions(mergeJoinWork);
+    }
+
     private boolean logExplainVectorization(BaseWork baseWork, String name) {
 
       if (!baseWork.getVectorizationExamined()) {
@@ -1101,6 +1171,13 @@ public class Vectorizer implements PhysicalPlanResolver {
       LOG.info("Reducer engine: " + reduceWork.getVectorReduceEngine());
     }
 
+    private void logMergeJoinWorkExplainVectorization(MergeJoinWork mergeJoinWork) {
+
+      if (!logExplainVectorization(mergeJoinWork, "MergeJoin")) {
+        return;
+      }
+    }
+
     private void convertMapWork(MapWork mapWork, boolean isTezOrSpark) throws SemanticException {
 
       // We have to evaluate the input format to see if vectorization is enabled, so
@@ -2319,7 +2396,7 @@ public class Vectorizer implements PhysicalPlanResolver {
         HiveConf.getVar(hiveConf,
             HiveConf.ConfVars.HIVE_TEST_VECTORIZATION_ENABLED_OVERRIDE);
     vectorizationEnabledOverride =
-        VectorizationEnabledOverride.nameMap.get(vectorizationEnabledOverrideString);
+        EnabledOverride.nameMap.get(vectorizationEnabledOverrideString);
 
     isVectorizationEnabled = HiveConf.getBoolVar(hiveConf,
         HiveConf.ConfVars.HIVE_VECTORIZATION_ENABLED);
@@ -3264,7 +3341,7 @@ public class Vectorizer implements PhysicalPlanResolver {
     HashTableImplementationType hashTableImplementationType = HashTableImplementationType.NONE;
     HashTableKind hashTableKind = HashTableKind.NONE;
     HashTableKeyType hashTableKeyType = HashTableKeyType.NONE;
-    VectorMapJoinVariation vectorMapJoinVariation = VectorMapJoinVariation.NONE;
+    VectorMapJoinVariation vectorMapJoinVariation = null;
 
     if (vectorDesc.getIsFastHashTableEnabled()) {
       hashTableImplementationType = HashTableImplementationType.FAST;
@@ -3334,6 +3411,10 @@ public class Vectorizer implements PhysicalPlanResolver {
       vectorMapJoinVariation = VectorMapJoinVariation.OUTER;
       hashTableKind = HashTableKind.HASH_MAP;
       break;
+    case JoinDesc.FULL_OUTER_JOIN:
+      vectorMapJoinVariation = VectorMapJoinVariation.FULL_OUTER;
+      hashTableKind = HashTableKind.HASH_MAP;
+      break;
     case JoinDesc.LEFT_SEMI_JOIN:
       vectorMapJoinVariation = VectorMapJoinVariation.LEFT_SEMI;
       hashTableKind = HashTableKind.HASH_SET;
@@ -3363,6 +3444,9 @@ public class Vectorizer implements PhysicalPlanResolver {
       case OUTER:
         opClass = VectorMapJoinOuterLongOperator.class;
         break;
+      case FULL_OUTER:
+        opClass = VectorMapJoinFullOuterLongOperator.class;
+        break;
       default:
         throw new HiveException("Unknown operator variation " + vectorMapJoinVariation);
       }
@@ -3381,6 +3465,9 @@ public class Vectorizer implements PhysicalPlanResolver {
       case OUTER:
         opClass = VectorMapJoinOuterStringOperator.class;
         break;
+      case FULL_OUTER:
+        opClass = VectorMapJoinFullOuterStringOperator.class;
+        break;
       default:
         throw new HiveException("Unknown operator variation " + vectorMapJoinVariation);
       }
@@ -3399,6 +3486,9 @@ public class Vectorizer implements PhysicalPlanResolver {
       case OUTER:
         opClass = VectorMapJoinOuterMultiKeyOperator.class;
         break;
+      case FULL_OUTER:
+        opClass = VectorMapJoinFullOuterMultiKeyOperator.class;
+        break;
       default:
         throw new HiveException("Unknown operator variation " + vectorMapJoinVariation);
       }
@@ -3414,6 +3504,10 @@ public class Vectorizer implements PhysicalPlanResolver {
     vectorDesc.setHashTableKind(hashTableKind);
     vectorDesc.setHashTableKeyType(hashTableKeyType);
     vectorDesc.setVectorMapJoinVariation(vectorMapJoinVariation);
+    if (vectorMapJoinVariation == VectorMapJoinVariation.FULL_OUTER) {
+
+      vectorDesc.setIsFullOuter(true);
+    }
     vectorDesc.setMinMaxEnabled(minMaxEnabled);
     vectorDesc.setVectorMapJoinInfo(vectorMapJoinInfo);
 
@@ -3526,6 +3620,8 @@ public class Vectorizer implements PhysicalPlanResolver {
     /*
      * Similarly, we need a mapping since a value expression can be a calculation and the value
      * will go into a scratch column.
+     *
+     * Value expressions include keys? YES.
      */
     int[] bigTableValueColumnMap = new int[allBigTableValueExpressions.length];
     String[] bigTableValueColumnNames = new String[allBigTableValueExpressions.length];
@@ -3565,18 +3661,24 @@ public class Vectorizer implements PhysicalPlanResolver {
     vectorDesc.setAllBigTableValueExpressions(allBigTableValueExpressions);
 
     /*
-     * Small table information.
+     * Column mapping.
      */
-    VectorColumnOutputMapping bigTableRetainedMapping =
-        new VectorColumnOutputMapping("Big Table Retained Mapping");
+    VectorColumnOutputMapping bigTableRetainMapping =
+        new VectorColumnOutputMapping("Big Table Retain Mapping");
 
-    VectorColumnOutputMapping bigTableOuterKeyMapping =
-        new VectorColumnOutputMapping("Big Table Outer Key Mapping");
+    VectorColumnOutputMapping nonOuterSmallTableKeyMapping =
+        new VectorColumnOutputMapping("Non Outer Small Table Key Key Mapping");
+
+    VectorColumnOutputMapping outerSmallTableKeyMapping =
+        new VectorColumnOutputMapping("Outer Small Table Key Mapping");
+
+    VectorColumnSourceMapping fullOuterSmallTableKeyMapping =
+        new VectorColumnSourceMapping("Full Outer Small Table Key Mapping");
 
     // The order of the fields in the LazyBinary small table value must be used, so
     // we use the source ordering flavor for the mapping.
-    VectorColumnSourceMapping smallTableMapping =
-        new VectorColumnSourceMapping("Small Table Mapping");
+    VectorColumnSourceMapping smallTableValueMapping =
+        new VectorColumnSourceMapping("Small Table Value Mapping");
 
     Byte[] order = desc.getTagOrder();
     Byte posSingleVectorMapJoinSmallTable = (order[0] == posBigTable ? order[1] : order[0]);
@@ -3586,7 +3688,6 @@ public class Vectorizer implements PhysicalPlanResolver {
      * Gather up big and small table output result information from the MapJoinDesc.
      */
     List<Integer> bigTableRetainList = desc.getRetainList().get(posBigTable);
-    int bigTableRetainSize = bigTableRetainList.size();
 
     int[] smallTableIndices;
     int smallTableIndicesSize;
@@ -3623,6 +3724,8 @@ public class Vectorizer implements PhysicalPlanResolver {
     VectorColumnSourceMapping projectionMapping = new VectorColumnSourceMapping("Projection Mapping");
 
     int nextOutputColumn = (order[0] == posBigTable ? 0 : smallTableResultSize);
+
+    final int bigTableRetainSize = bigTableRetainList.size();
     for (int i = 0; i < bigTableRetainSize; i++) {
 
       // Since bigTableValueExpressions may do a calculation and produce a scratch column, we
@@ -3636,9 +3739,10 @@ public class Vectorizer implements PhysicalPlanResolver {
       projectionMapping.add(nextOutputColumn, batchColumnIndex, typeInfo);
 
       // Collect columns we copy from the big table batch to the overflow batch.
-      if (!bigTableRetainedMapping.containsOutputColumn(batchColumnIndex)) {
+      if (!bigTableRetainMapping.containsOutputColumn(batchColumnIndex)) {
+
         // Tolerate repeated use of a big table column.
-        bigTableRetainedMapping.add(batchColumnIndex, batchColumnIndex, typeInfo);
+        bigTableRetainMapping.add(batchColumnIndex, batchColumnIndex, typeInfo);
       }
 
       nextOutputColumn++;
@@ -3655,10 +3759,8 @@ public class Vectorizer implements PhysicalPlanResolver {
     nextOutputColumn = firstSmallTableOutputColumn;
 
     // Small table indices has more information (i.e. keys) than retain, so use it if it exists...
-    String[] bigTableRetainedNames;
     if (smallTableIndicesSize > 0) {
       smallTableOutputCount = smallTableIndicesSize;
-      bigTableRetainedNames = new String[smallTableOutputCount];
 
       for (int i = 0; i < smallTableIndicesSize; i++) {
         if (smallTableIndices[i] >= 0) {
@@ -3670,8 +3772,7 @@ public class Vectorizer implements PhysicalPlanResolver {
 
           // Since bigTableKeyExpressions may do a calculation and produce a scratch column, we
           // need to map the right column.
-          int batchKeyColumn = bigTableKeyColumnMap[keyIndex];
-          bigTableRetainedNames[i] = bigTableKeyColumnNames[keyIndex];
+          int bigTableKeyColumn = bigTableKeyColumnMap[keyIndex];
           TypeInfo typeInfo = bigTableKeyTypeInfos[keyIndex];
 
           if (!isOuterJoin) {
@@ -3679,25 +3780,30 @@ public class Vectorizer implements PhysicalPlanResolver {
             // Optimize inner join keys of small table results.
 
             // Project the big table key into the small table result "area".
-            projectionMapping.add(nextOutputColumn, batchKeyColumn, typeInfo);
+            projectionMapping.add(nextOutputColumn, bigTableKeyColumn, typeInfo);
 
-            if (!bigTableRetainedMapping.containsOutputColumn(batchKeyColumn)) {
-              // If necessary, copy the big table key into the overflow batch's small table
-              // result "area".
-              bigTableRetainedMapping.add(batchKeyColumn, batchKeyColumn, typeInfo);
+            if (!bigTableRetainMapping.containsOutputColumn(bigTableKeyColumn)) {
+
+              // When the Big Key is not retained in the output result, we do need to copy the
+              // Big Table key into the overflow batch so the projection of it (Big Table key) to
+              // the Small Table key will work properly...
+              //
+              nonOuterSmallTableKeyMapping.add(bigTableKeyColumn, bigTableKeyColumn, typeInfo);
             }
           } else {
 
-            // For outer joins, since the small table key can be null when there is no match,
+            // For outer joins, since the small table key can be null when there for NOMATCH,
             // we must have a physical (scratch) column for those keys.  We cannot use the
-            // projection optimization used by inner joins above.
+            // projection optimization used by non-[FULL} OUTER joins above.
 
             int scratchColumn = vContext.allocateScratchColumn(typeInfo);
             projectionMapping.add(nextOutputColumn, scratchColumn, typeInfo);
 
-            bigTableRetainedMapping.add(batchKeyColumn, scratchColumn, typeInfo);
+            outerSmallTableKeyMapping.add(bigTableKeyColumn, scratchColumn, typeInfo);
 
-            bigTableOuterKeyMapping.add(batchKeyColumn, scratchColumn, typeInfo);
+            // For FULL OUTER MapJoin, we need to be able to deserialize a Small Table key
+            // into the output result.
+            fullOuterSmallTableKeyMapping.add(keyIndex, scratchColumn, typeInfo);
           }
         } else {
 
@@ -3711,21 +3817,18 @@ public class Vectorizer implements PhysicalPlanResolver {
             smallTableExprVectorizes = false;
           }
 
-          bigTableRetainedNames[i] = smallTableExprNode.toString();
-
           TypeInfo typeInfo = smallTableExprNode.getTypeInfo();
 
           // Make a new big table scratch column for the small table value.
           int scratchColumn = vContext.allocateScratchColumn(typeInfo);
           projectionMapping.add(nextOutputColumn, scratchColumn, typeInfo);
 
-          smallTableMapping.add(smallTableValueIndex, scratchColumn, typeInfo);
+          smallTableValueMapping.add(smallTableValueIndex, scratchColumn, typeInfo);
         }
         nextOutputColumn++;
       }
     } else if (smallTableRetainSize > 0) {
       smallTableOutputCount = smallTableRetainSize;
-      bigTableRetainedNames = new String[smallTableOutputCount];
 
       // Only small table values appear in join output result.
 
@@ -3738,21 +3841,24 @@ public class Vectorizer implements PhysicalPlanResolver {
           smallTableExprVectorizes = false;
         }
 
-        bigTableRetainedNames[i] = smallTableExprNode.toString();
-
         // Make a new big table scratch column for the small table value.
         TypeInfo typeInfo = smallTableExprNode.getTypeInfo();
         int scratchColumn = vContext.allocateScratchColumn(typeInfo);
 
         projectionMapping.add(nextOutputColumn, scratchColumn, typeInfo);
 
-        smallTableMapping.add(smallTableValueIndex, scratchColumn, typeInfo);
+        smallTableValueMapping.add(smallTableValueIndex, scratchColumn, typeInfo);
         nextOutputColumn++;
       }
-    } else {
-      bigTableRetainedNames = new String[0];
     }
 
+    Map<Byte, List<ExprNodeDesc>> filterExpressions = desc.getFilters();
+    VectorExpression[] bigTableFilterExpressions =
+        vContext.getVectorExpressions(
+            filterExpressions.get(posBigTable),
+            VectorExpressionDescriptor.Mode.FILTER);
+    vectorMapJoinInfo.setBigTableFilterExpressions(bigTableFilterExpressions);
+
     boolean useOptimizedTable =
         HiveConf.getBoolVar(hiveConf, HiveConf.ConfVars.HIVEMAPJOINUSEOPTIMIZEDTABLE);
 
@@ -3808,15 +3914,23 @@ public class Vectorizer implements PhysicalPlanResolver {
 
     // Convert dynamic arrays and maps to simple arrays.
 
-    bigTableRetainedMapping.finalize();
+    bigTableRetainMapping.finalize();
+    vectorMapJoinInfo.setBigTableRetainColumnMap(bigTableRetainMapping.getOutputColumns());
+    vectorMapJoinInfo.setBigTableRetainTypeInfos(bigTableRetainMapping.getTypeInfos());
+
+    nonOuterSmallTableKeyMapping.finalize();
+    vectorMapJoinInfo.setNonOuterSmallTableKeyColumnMap(nonOuterSmallTableKeyMapping.getOutputColumns());
+    vectorMapJoinInfo.setNonOuterSmallTableKeyTypeInfos(nonOuterSmallTableKeyMapping.getTypeInfos());
+
+    outerSmallTableKeyMapping.finalize();
+    fullOuterSmallTableKeyMapping.finalize();
 
-    bigTableOuterKeyMapping.finalize();
+    vectorMapJoinInfo.setOuterSmallTableKeyMapping(outerSmallTableKeyMapping);
+    vectorMapJoinInfo.setFullOuterSmallTableKeyMapping(fullOuterSmallTableKeyMapping);
 
-    smallTableMapping.finalize();
+    smallTableValueMapping.finalize();
 
-    vectorMapJoinInfo.setBigTableRetainedMapping(bigTableRetainedMapping);
-    vectorMapJoinInfo.setBigTableOuterKeyMapping(bigTableOuterKeyMapping);
-    vectorMapJoinInfo.setSmallTableMapping(smallTableMapping);
+    vectorMapJoinInfo.setSmallTableValueMapping(smallTableValueMapping);
 
     projectionMapping.finalize();
 

http://git-wip-us.apache.org/repos/asf/hive/blob/a37827ec/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkMapJoinOptimizer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkMapJoinOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkMapJoinOptimizer.java
index 689c888..ad6db21 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkMapJoinOptimizer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/spark/SparkMapJoinOptimizer.java
@@ -459,6 +459,9 @@ public class SparkMapJoinOptimizer implements NodeProcessor {
         MapJoinProcessor.convertJoinOpMapJoinOp(context.getConf(), joinOp,
             joinOp.getConf().isLeftInputJoin(), joinOp.getConf().getBaseSrc(),
             joinOp.getConf().getMapAliases(), bigTablePosition, true);
+    if (mapJoinOp == null) {
+      return null;
+    }
 
     Operator<? extends OperatorDesc> parentBigTableOp =
         mapJoinOp.getParentOperators().get(bigTablePosition);

http://git-wip-us.apache.org/repos/asf/hive/blob/a37827ec/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainConfiguration.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainConfiguration.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainConfiguration.java
index 2f12bda..a92502e 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainConfiguration.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainConfiguration.java
@@ -49,6 +49,7 @@ public class ExplainConfiguration {
   private VectorizationDetailLevel vectorizationDetailLevel = VectorizationDetailLevel.SUMMARY;
   private boolean locks = false;
   private boolean ast = false;
+  private boolean debug = false;
 
   private Path explainRootPath;
   private Map<String, Long> opIdToRuntimeNumRows;
@@ -139,6 +140,14 @@ public class ExplainConfiguration {
     this.vectorizationDetailLevel = vectorizationDetailLevel;
   }
 
+  public boolean isDebug() {
+    return debug;
+  }
+
+  public void setDebug(boolean debug) {
+    this.debug = debug;
+  }
+
   public Path getExplainRootPath() {
     return explainRootPath;
   }

http://git-wip-us.apache.org/repos/asf/hive/blob/a37827ec/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java
index b17efff..428a3c9 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ExplainSemanticAnalyzer.java
@@ -116,6 +116,8 @@ public class ExplainSemanticAnalyzer extends BaseSemanticAnalyzer {
         config.setLocks(true);
       } else if (explainOptions == HiveParser.KW_AST){
         config.setAst(true);
+      } else if (explainOptions == HiveParser.KW_DEBUG) {
+        config.setDebug(true);
       } else {
         // UNDONE: UNKNOWN OPTION?
       }

http://git-wip-us.apache.org/repos/asf/hive/blob/a37827ec/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g b/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g
index 807f6f5..3caa51f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveLexer.g
@@ -189,6 +189,7 @@ KW_FILE: 'FILE';
 KW_JAR: 'JAR';
 KW_EXPLAIN: 'EXPLAIN';
 KW_EXTENDED: 'EXTENDED';
+KW_DEBUG: 'DEBUG';
 KW_FORMATTED: 'FORMATTED';
 KW_DEPENDENCY: 'DEPENDENCY';
 KW_LOGICAL: 'LOGICAL';

http://git-wip-us.apache.org/repos/asf/hive/blob/a37827ec/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g b/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g
index 2cfcdcc..48f7303 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g
@@ -569,6 +569,7 @@ import org.apache.hadoop.hive.conf.HiveConf;
     xlateMap.put("KW_FUNCTION", "FUNCTION");
     xlateMap.put("KW_EXPLAIN", "EXPLAIN");
     xlateMap.put("KW_EXTENDED", "EXTENDED");
+    xlateMap.put("KW_DEBUG", "DEBUG");
     xlateMap.put("KW_SERDE", "SERDE");
     xlateMap.put("KW_WITH", "WITH");
     xlateMap.put("KW_SERDEPROPERTIES", "SERDEPROPERTIES");
@@ -810,6 +811,7 @@ explainOption
     | KW_LOCKS
     | KW_AST
     | (KW_VECTORIZATION vectorizationOnly? vectorizatonDetail?)
+    | KW_DEBUG
     ;
 
 vectorizationOnly

http://git-wip-us.apache.org/repos/asf/hive/blob/a37827ec/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g b/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g
index f327a12..f9c97e0 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/IdentifiersParser.g
@@ -831,6 +831,7 @@ nonReserved
     | KW_OPERATOR
     | KW_EXPRESSION
     | KW_DETAIL
+    | KW_DEBUG
     | KW_WAIT
     | KW_ZONE
     | KW_TIMESTAMPTZ

http://git-wip-us.apache.org/repos/asf/hive/blob/a37827ec/ql/src/java/org/apache/hadoop/hive/ql/plan/Explain.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/Explain.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/Explain.java
index da30243..030bb61 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/Explain.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/Explain.java
@@ -28,7 +28,7 @@ import java.lang.annotation.RetentionPolicy;
 @Retention(RetentionPolicy.RUNTIME)
 public @interface Explain {
   public enum Level {
-    USER, DEFAULT, EXTENDED;
+    USER, DEFAULT, EXTENDED, DEBUG;
     public boolean in(Level[] levels) {
       for (Level level : levels) {
         if (level.equals(this)) {

http://git-wip-us.apache.org/repos/asf/hive/blob/a37827ec/ql/src/java/org/apache/hadoop/hive/ql/plan/ExplainWork.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ExplainWork.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/ExplainWork.java
index 0654f57..01da4d5 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/ExplainWork.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/ExplainWork.java
@@ -165,6 +165,10 @@ public class ExplainWork implements Serializable {
     return config.getVectorizationDetailLevel();
   }
 
+  public boolean isDebug() {
+    return config.isDebug();
+  }
+
   public ParseContext getParseContext() {
     return pCtx;
   }

http://git-wip-us.apache.org/repos/asf/hive/blob/a37827ec/ql/src/java/org/apache/hadoop/hive/ql/plan/JoinCondDesc.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/JoinCondDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/JoinCondDesc.java
index 3820d57..0eb03e9 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/JoinCondDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/JoinCondDesc.java
@@ -126,7 +126,7 @@ public class JoinCondDesc implements Serializable {
       sb.append("Inner Join ");
       break;
     case JoinDesc.FULL_OUTER_JOIN:
-      sb.append("Outer Join ");
+      sb.append("Full Outer Join ");
       break;
     case JoinDesc.LEFT_OUTER_JOIN:
       sb.append("Left Outer Join ");

http://git-wip-us.apache.org/repos/asf/hive/blob/a37827ec/ql/src/java/org/apache/hadoop/hive/ql/plan/JoinDesc.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/JoinDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/JoinDesc.java
index 4313a6b..2c93c2a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/JoinDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/JoinDesc.java
@@ -291,7 +291,7 @@ public class JoinDesc extends AbstractOperatorDesc {
 
           first = false;
           sb.append("{");
-          sb.append(expr.getExprString());
+          sb.append(expr == null ? "NULL" : expr.getExprString());
           sb.append("}");
         }
       }

http://git-wip-us.apache.org/repos/asf/hive/blob/a37827ec/ql/src/java/org/apache/hadoop/hive/ql/plan/MapJoinDesc.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/MapJoinDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/MapJoinDesc.java
index dc4f085..8ba5101 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/MapJoinDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/MapJoinDesc.java
@@ -33,10 +33,16 @@ import java.util.Set;
 
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.ql.exec.MemoryMonitorInfo;
+import org.apache.hadoop.hive.ql.exec.persistence.MapJoinObjectSerDeContext;
+import org.apache.hadoop.hive.ql.optimizer.signature.Signature;
 import org.apache.hadoop.hive.ql.plan.Explain.Level;
 import org.apache.hadoop.hive.ql.plan.Explain.Vectorization;
 import org.apache.hadoop.hive.ql.plan.VectorMapJoinDesc.HashTableImplementationType;
 import org.apache.hadoop.hive.ql.plan.VectorMapJoinDesc.VectorMapJoinVariation;
+import org.apache.hadoop.hive.serde2.AbstractSerDe;
+import org.apache.hadoop.hive.serde2.SerDeException;
+import org.apache.hadoop.hive.serde2.SerDeUtils;
+import org.apache.hive.common.util.ReflectionUtil;
 
 /**
  * Map Join operator Descriptor implementation.
@@ -91,6 +97,7 @@ public class MapJoinDesc extends JoinDesc implements Serializable {
     this.keys = clone.keys;
     this.keyTblDesc = clone.keyTblDesc;
     this.valueTblDescs = clone.valueTblDescs;
+    this.valueFilteredTblDescs = clone.valueFilteredTblDescs;
     this.posBigTable = clone.posBigTable;
     this.valueIndices = clone.valueIndices;
     this.retainList = clone.retainList;
@@ -208,6 +215,16 @@ public class MapJoinDesc extends JoinDesc implements Serializable {
     this.dumpFilePrefix = dumpFilePrefix;
   }
 
+  // NOTE: Debugging only.
+  @Explain(displayName = "keyExpressions", explainLevels = { Level.DEBUG })
+  public Map<Byte, String> getKeyExpressionString() {
+    Map<Byte, String> keyMap = new LinkedHashMap<Byte, String>();
+    for (Map.Entry<Byte, List<ExprNodeDesc>> k: getKeys().entrySet()) {
+      keyMap.put(k.getKey(), k.getValue().toString());
+    }
+    return keyMap;
+  }
+
   /**
    * @return the keys in string form
    */
@@ -292,6 +309,60 @@ public class MapJoinDesc extends JoinDesc implements Serializable {
     return valueTblDescs;
   }
 
+  // NOTE: Debugging only.
+  @Explain(displayName = "keyContext", explainLevels = { Level.DEBUG })
+  public String getDebugKeyContext() {
+    MapJoinObjectSerDeContext keyContext;
+    try {
+      AbstractSerDe keySerde =
+          (AbstractSerDe) ReflectionUtil.newInstance(
+              keyTblDesc.getDeserializerClass(), null);
+      SerDeUtils.initializeSerDe(keySerde, null, keyTblDesc.getProperties(), null);
+      keyContext = new MapJoinObjectSerDeContext(keySerde, false);
+    } catch (SerDeException e) {
+      return null;
+    }
+    return keyContext.stringify();
+  }
+
+  private boolean hasFilter(int alias, int[][] filterMaps) {
+    return filterMaps != null && filterMaps[alias] != null;
+  }
+
+  // NOTE: Debugging only.
+  @Explain(displayName = "valueContexts", explainLevels = { Level.DEBUG })
+  public String getDebugValueContext() {
+    List<String> valueContextStringList = new ArrayList<String>();
+    try {
+      boolean noOuterJoin = getNoOuterJoin();
+      // Order in which the results should be output.
+      Byte[] order = getTagOrder();
+      int[][] filterMaps = getFilterMap();
+
+      for (int pos = 0; pos < order.length; pos++) {
+        if (pos == posBigTable) {
+          continue;
+        }
+        TableDesc valueTableDesc;
+        if (noOuterJoin) {
+          valueTableDesc = getValueTblDescs().get(pos);
+        } else {
+          valueTableDesc = getValueFilteredTblDescs().get(pos);
+        }
+        AbstractSerDe valueSerDe =
+            (AbstractSerDe) ReflectionUtil.newInstance(
+                valueTableDesc.getDeserializerClass(), null);
+        SerDeUtils.initializeSerDe(valueSerDe, null, valueTableDesc.getProperties(), null);
+        MapJoinObjectSerDeContext valueContext =
+            new MapJoinObjectSerDeContext(valueSerDe, hasFilter(pos, filterMaps));
+        valueContextStringList.add(pos + ":" + valueContext.stringify());
+      }
+    } catch (SerDeException e) {
+      return null;
+    }
+    return valueContextStringList.toString();
+  }
+
   /**
    * @param valueTblDescs
    *          the valueTblDescs to set
@@ -378,6 +449,8 @@ public class MapJoinDesc extends JoinDesc implements Serializable {
     return genJoinKeys;
   }
 
+  @Explain(displayName = "DynamicPartitionHashJoin", explainLevels = { Level.USER, Level.DEFAULT,
+      Level.EXTENDED }, displayOnlyOnTrue = true)
   public boolean isDynamicPartitionHashJoin() {
     return isDynamicPartitionHashJoin;
   }
@@ -386,6 +459,25 @@ public class MapJoinDesc extends JoinDesc implements Serializable {
     this.isDynamicPartitionHashJoin = isDistributedHashJoin;
   }
 
+  // NOTE: Debugging only.
+  @Explain(displayName = "outer filter mappings", explainLevels = { Level.DEBUG })
+  public String getDebugOuterFilterMapString() {
+    if (conds.length != 1) {
+      return null;
+    }
+    JoinCondDesc cond = conds[0];
+    if (cond.getType() != JoinDesc.FULL_OUTER_JOIN &&
+        cond.getType() != JoinDesc.LEFT_OUTER_JOIN &&
+        cond.getType() != JoinDesc.RIGHT_OUTER_JOIN) {
+      return null;
+    }
+    int[][] fm = getFilterMap();
+    if (fm == null) {
+      return null;
+    }
+    return Arrays.deepToString(fm);
+  }
+
   // Use LinkedHashSet to give predictable display order.
   private static final Set<String> vectorizableMapJoinNativeEngines =
       new LinkedHashSet<String>(Arrays.asList("tez", "spark"));
@@ -401,7 +493,9 @@ public class MapJoinDesc extends JoinDesc implements Serializable {
     public MapJoinOperatorExplainVectorization(MapJoinDesc mapJoinDesc,
         VectorMapJoinDesc vectorMapJoinDesc) {
       // VectorMapJoinOperator is not native vectorized.
-      super(vectorMapJoinDesc, vectorMapJoinDesc.getHashTableImplementationType() != HashTableImplementationType.NONE);
+      super(
+          vectorMapJoinDesc,
+          vectorMapJoinDesc.getHashTableImplementationType() != HashTableImplementationType.NONE);
       this.mapJoinDesc = mapJoinDesc;
       this.vectorMapJoinDesc = vectorMapJoinDesc;
       vectorMapJoinInfo =
@@ -414,7 +508,8 @@ public class MapJoinDesc extends JoinDesc implements Serializable {
 
       String engine = vectorMapJoinDesc.getEngine();
       String engineInSupportedCondName =
-          HiveConf.ConfVars.HIVE_EXECUTION_ENGINE.varname + " " + engine + " IN " + vectorizableMapJoinNativeEngines;
+          HiveConf.ConfVars.HIVE_EXECUTION_ENGINE.varname + " " +
+              engine + " IN " + vectorizableMapJoinNativeEngines;
       boolean engineInSupported = vectorizableMapJoinNativeEngines.contains(engine);
 
       boolean isFastHashTableEnabled = vectorMapJoinDesc.getIsFastHashTableEnabled();
@@ -469,7 +564,8 @@ public class MapJoinDesc extends JoinDesc implements Serializable {
       return conditions;
     }
 
-    @Explain(vectorization = Vectorization.OPERATOR, displayName = "nativeConditionsMet", explainLevels = { Level.DEFAULT, Level.EXTENDED })
+    @Explain(vectorization = Vectorization.OPERATOR, displayName = "nativeConditionsMet",
+        explainLevels = { Level.DEFAULT, Level.EXTENDED })
     public List<String> getNativeConditionsMet() {
       if (nativeConditions == null) {
         nativeConditions = createNativeConditions();
@@ -477,7 +573,8 @@ public class MapJoinDesc extends JoinDesc implements Serializable {
       return VectorizationCondition.getConditionsMet(nativeConditions);
     }
 
-    @Explain(vectorization = Vectorization.OPERATOR, displayName = "nativeConditionsNotMet", explainLevels = { Level.DEFAULT, Level.EXTENDED })
+    @Explain(vectorization = Vectorization.OPERATOR, displayName = "nativeConditionsNotMet",
+        explainLevels = { Level.DEFAULT, Level.EXTENDED })
     public List<String> getNativeConditionsNotMet() {
       if (nativeConditions == null) {
         nativeConditions = createNativeConditions();
@@ -485,7 +582,8 @@ public class MapJoinDesc extends JoinDesc implements Serializable {
       return VectorizationCondition.getConditionsNotMet(nativeConditions);
     }
 
-    @Explain(vectorization = Vectorization.EXPRESSION, displayName = "bigTableKeyExpressions", explainLevels = { Level.DEFAULT, Level.EXTENDED })
+    @Explain(vectorization = Vectorization.OPERATOR, displayName = "bigTableKeyExpressions",
+        explainLevels = { Level.DEFAULT, Level.EXTENDED })
     public List<String> getBigTableKeyExpressions() {
       return vectorExpressionsToStringList(
           isNative ?
@@ -493,8 +591,18 @@ public class MapJoinDesc extends JoinDesc implements Serializable {
               vectorMapJoinDesc.getAllBigTableKeyExpressions());
     }
 
-    @Explain(vectorization = Vectorization.DETAIL, displayName = "bigTableKeyColumnNums", explainLevels = { Level.DEFAULT, Level.EXTENDED })
-    public String getBigTableKeyColumnNums() {
+    @Explain(vectorization = Vectorization.EXPRESSION, displayName = "hashTableImplementationType",
+        explainLevels = { Level.DEFAULT, Level.EXTENDED })
+    public String hashTableImplementationType() {
+      if (!isNative) {
+        return null;
+      }
+      return vectorMapJoinDesc.getHashTableImplementationType().name();
+    }
+
+    @Explain(vectorization = Vectorization.DETAIL, displayName = "bigTableKeyColumns",
+        explainLevels = { Level.DEFAULT, Level.EXTENDED })
+    public List<String> getBigTableKeyColumns() {
       if (!isNative) {
         return null;
       }
@@ -502,10 +610,13 @@ public class MapJoinDesc extends JoinDesc implements Serializable {
       if (bigTableKeyColumnMap.length == 0) {
         return null;
       }
-      return Arrays.toString(bigTableKeyColumnMap);
+      return outputColumnsAndTypesToStringList(
+          vectorMapJoinInfo.getBigTableKeyColumnMap(),
+          vectorMapJoinInfo.getBigTableKeyTypeInfos());
     }
 
-    @Explain(vectorization = Vectorization.EXPRESSION, displayName = "bigTableValueExpressions", explainLevels = { Level.DEFAULT, Level.EXTENDED })
+    @Explain(vectorization = Vectorization.EXPRESSION, displayName = "bigTableValueExpressions",
+        explainLevels = { Level.DEFAULT, Level.EXTENDED })
     public List<String> getBigTableValueExpressions() {
       return vectorExpressionsToStringList(
           isNative ?
@@ -513,8 +624,18 @@ public class MapJoinDesc extends JoinDesc implements Serializable {
               vectorMapJoinDesc.getAllBigTableValueExpressions());
     }
 
-    @Explain(vectorization = Vectorization.DETAIL, displayName = "bigTableValueColumnNums", explainLevels = { Level.DEFAULT, Level.EXTENDED })
-    public String getBigTableValueColumnNums() {
+    @Explain(vectorization = Vectorization.EXPRESSION, displayName = "bigTableFilterExpressions",
+        explainLevels = { Level.DEFAULT, Level.EXTENDED })
+    public List<String> getBigTableFilterExpressions() {
+      if (!isNative) {
+        return null;
+      }
+      return vectorExpressionsToStringList(vectorMapJoinInfo.getBigTableFilterExpressions());
+    }
+
+    @Explain(vectorization = Vectorization.DETAIL, displayName = "bigTableValueColumns",
+        explainLevels = { Level.DEFAULT, Level.EXTENDED })
+    public List<String> getBigTableValueColumns() {
       if (!isNative) {
         return null;
       }
@@ -522,48 +643,78 @@ public class MapJoinDesc extends JoinDesc implements Serializable {
       if (bigTableValueColumnMap.length == 0) {
         return null;
       }
-      return Arrays.toString(bigTableValueColumnMap);
+      return outputColumnsAndTypesToStringList(
+          vectorMapJoinInfo.getBigTableValueColumnMap(),
+          vectorMapJoinInfo.getBigTableValueTypeInfos());
     }
 
-    @Explain(vectorization = Vectorization.DETAIL, displayName = "smallTableMapping", explainLevels = { Level.DEFAULT, Level.EXTENDED })
-    public String getSmallTableColumns() {
+    @Explain(vectorization = Vectorization.DETAIL, displayName = "smallTableValueMapping",
+        explainLevels = { Level.DEFAULT, Level.EXTENDED })
+    public List<String> getSmallTableColumns() {
       if (!isNative) {
         return null;
       }
-      return outputColumnsToStringList(vectorMapJoinInfo.getSmallTableMapping());
+      return outputColumnsAndTypesToStringList(vectorMapJoinInfo.getSmallTableValueMapping());
     }
 
-    @Explain(vectorization = Vectorization.DETAIL, displayName = "projectedOutputColumnNums", explainLevels = { Level.DEFAULT, Level.EXTENDED })
-    public String getProjectedOutputColumnNums() {
+    @Explain(vectorization = Vectorization.DETAIL, displayName = "projectedOutput",
+        explainLevels = { Level.DEFAULT, Level.EXTENDED })
+    public List<String> getProjectedOutputColumnNums() {
       if (!isNative) {
         return null;
       }
-      return outputColumnsToStringList(vectorMapJoinInfo.getProjectionMapping());
+      return outputColumnsAndTypesToStringList(vectorMapJoinInfo.getProjectionMapping());
     }
 
-    @Explain(vectorization = Vectorization.DETAIL, displayName = "bigTableOuterKeyMapping", explainLevels = { Level.DEFAULT, Level.EXTENDED })
-    public List<String> getBigTableOuterKey() {
-      if (!isNative || vectorMapJoinDesc.getVectorMapJoinVariation() != VectorMapJoinVariation.OUTER) {
+    @Explain(vectorization = Vectorization.DETAIL, displayName = "bigTableRetainColumnNums",
+        explainLevels = { Level.DEFAULT, Level.EXTENDED })
+    public String getBigTableRetainedColumnNums() {
+      if (!isNative) {
         return null;
       }
-      return columnMappingToStringList(vectorMapJoinInfo.getBigTableOuterKeyMapping());
+      return Arrays.toString(vectorMapJoinInfo.getBigTableRetainColumnMap());
     }
 
-    @Explain(vectorization = Vectorization.DETAIL, displayName = "bigTableRetainedColumnNums", explainLevels = { Level.DEFAULT, Level.EXTENDED })
-    public String getBigTableRetainedColumnNums() {
-      if (!isNative) {
+    @Explain(vectorization = Vectorization.DETAIL, displayName = "nonOuterSmallTableKeyMapping",
+        explainLevels = { Level.DEFAULT, Level.EXTENDED })
+    public String getNonOuterSmallTableKeyMapping() {
+      if (!isNative ||
+          (vectorMapJoinDesc.getVectorMapJoinVariation() == VectorMapJoinVariation.OUTER ||
+          vectorMapJoinDesc.getVectorMapJoinVariation() == VectorMapJoinVariation.FULL_OUTER)) {
+        return null;
+      }
+      return Arrays.toString(vectorMapJoinInfo.getNonOuterSmallTableKeyColumnMap());
+    }
+
+    @Explain(vectorization = Vectorization.DETAIL, displayName = "outerSmallTableKeyMapping",
+        explainLevels = { Level.DEFAULT, Level.EXTENDED })
+    public List<String> getOuterSmallTableKeyMapping() {
+      if (!isNative ||
+          vectorMapJoinDesc.getVectorMapJoinVariation() != VectorMapJoinVariation.OUTER) {
+        return null;
+      }
+      return columnMappingToStringList(vectorMapJoinInfo.getOuterSmallTableKeyMapping());
+    }
+
+    @Explain(vectorization = Vectorization.DETAIL, displayName = "fullOuterSmallTableKeyMapping",
+        explainLevels = { Level.DEFAULT, Level.EXTENDED })
+    public List<String> getFullOuterSmallTableKeyMapping() {
+      if (!isNative ||
+          vectorMapJoinDesc.getVectorMapJoinVariation() != VectorMapJoinVariation.FULL_OUTER) {
         return null;
       }
-      return outputColumnsToStringList(vectorMapJoinInfo.getBigTableRetainedMapping());
+      return columnMappingToStringList(vectorMapJoinInfo.getFullOuterSmallTableKeyMapping());
     }
 
-    @Explain(vectorization = Vectorization.OPERATOR, displayName = "nativeNotSupportedKeyTypes", explainLevels = { Level.DEFAULT, Level.EXTENDED })
+    @Explain(vectorization = Vectorization.OPERATOR, displayName = "nativeNotSupportedKeyTypes",
+        explainLevels = { Level.DEFAULT, Level.EXTENDED })
     public List<String> getNativeNotSupportedKeyTypes() {
       return vectorMapJoinDesc.getNotSupportedKeyTypes();
     }
   }
 
-  @Explain(vectorization = Vectorization.OPERATOR, displayName = "Map Join Vectorization", explainLevels = { Level.DEFAULT, Level.EXTENDED })
+  @Explain(vectorization = Vectorization.OPERATOR, displayName = "Map Join Vectorization",
+      explainLevels = { Level.DEFAULT, Level.EXTENDED })
   public MapJoinOperatorExplainVectorization getMapJoinVectorization() {
     VectorMapJoinDesc vectorMapJoinDesc = (VectorMapJoinDesc) getVectorDesc();
     if (vectorMapJoinDesc == null || this instanceof SMBJoinDesc) {
@@ -587,7 +738,8 @@ public class MapJoinDesc extends JoinDesc implements Serializable {
   }
 
   // Handle dual nature.
-  @Explain(vectorization = Vectorization.OPERATOR, displayName = "SMB Map Join Vectorization", explainLevels = { Level.DEFAULT, Level.EXTENDED })
+  @Explain(vectorization = Vectorization.OPERATOR, displayName = "SMB Map Join Vectorization",
+      explainLevels = { Level.DEFAULT, Level.EXTENDED })
   public SMBJoinOperatorExplainVectorization getSMBJoinVectorization() {
     VectorSMBJoinDesc vectorSMBJoinDesc = (VectorSMBJoinDesc) getVectorDesc();
     if (vectorSMBJoinDesc == null || !(this instanceof SMBJoinDesc)) {

http://git-wip-us.apache.org/repos/asf/hive/blob/a37827ec/ql/src/java/org/apache/hadoop/hive/ql/plan/MergeJoinWork.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/MergeJoinWork.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/MergeJoinWork.java
index 45f3347..dd907ef 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/MergeJoinWork.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/MergeJoinWork.java
@@ -29,7 +29,9 @@ import org.apache.hadoop.hive.ql.exec.CommonMergeJoinOperator;
 import org.apache.hadoop.hive.ql.exec.HashTableDummyOperator;
 import org.apache.hadoop.hive.ql.exec.Operator;
 import org.apache.hadoop.hive.ql.exec.ReduceSinkOperator;
+import org.apache.hadoop.hive.ql.plan.BaseWork.BaseExplainVectorization;
 import org.apache.hadoop.hive.ql.plan.Explain.Level;
+import org.apache.hadoop.hive.ql.plan.Explain.Vectorization;
 import org.apache.hadoop.mapred.JobConf;
 
 public class MergeJoinWork extends BaseWork {
@@ -180,4 +182,59 @@ public class MergeJoinWork extends BaseWork {
   public void addDummyOp(HashTableDummyOperator dummyOp) {
     getMainWork().addDummyOp(dummyOp);
   }
+
+  /**
+   * For now, this class just says in EXPLAIN VECTORIZATION we don't support vectorization of the
+   * Merge Join vertex instead of being silent about it.
+   */
+  public class MergeJoinExplainVectorization extends BaseExplainVectorization {
+
+    private final MergeJoinWork mergeJoinWork;
+
+    private VectorizationCondition[] mergeWorkVectorizationConditions;
+
+    public MergeJoinExplainVectorization(MergeJoinWork mergeJoinWork) {
+      super(mergeJoinWork);
+      this.mergeJoinWork = mergeJoinWork;
+    }
+
+    private VectorizationCondition[] createMergeWorkExplainVectorizationConditions() {
+
+      boolean enabled = false;
+
+      VectorizationCondition[] conditions = new VectorizationCondition[] {
+          new VectorizationCondition(
+              enabled,
+              "Vectorizing MergeJoin Supported")
+      };
+      return conditions;
+    }
+
+    @Explain(vectorization = Vectorization.SUMMARY, displayName = "enableConditionsMet",
+        explainLevels = { Level.DEFAULT, Level.EXTENDED })
+    public List<String> getEnableConditionsMet() {
+      if (mergeWorkVectorizationConditions == null) {
+        mergeWorkVectorizationConditions = createMergeWorkExplainVectorizationConditions();
+      }
+      return VectorizationCondition.getConditionsMet(mergeWorkVectorizationConditions);
+    }
+
+    @Explain(vectorization = Vectorization.SUMMARY, displayName = "enableConditionsNotMet",
+        explainLevels = { Level.DEFAULT, Level.EXTENDED })
+    public List<String> getEnableConditionsNotMet() {
+      if (mergeWorkVectorizationConditions == null) {
+        mergeWorkVectorizationConditions = createMergeWorkExplainVectorizationConditions();
+      }
+      return VectorizationCondition.getConditionsNotMet(mergeWorkVectorizationConditions);
+    }
+  }
+
+  @Explain(vectorization = Vectorization.SUMMARY, displayName = "MergeJoin Vectorization",
+      explainLevels = { Level.DEFAULT, Level.EXTENDED })
+  public MergeJoinExplainVectorization getReduceExplainVectorization() {
+    if (!getVectorizationExamined()) {
+      return null;
+    }
+    return new MergeJoinExplainVectorization(this);
+  }
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/a37827ec/ql/src/java/org/apache/hadoop/hive/ql/plan/OperatorExplainVectorization.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/OperatorExplainVectorization.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/OperatorExplainVectorization.java
index 446b810..5439e14 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/OperatorExplainVectorization.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/OperatorExplainVectorization.java
@@ -59,6 +59,25 @@ public class OperatorExplainVectorization {
     return Arrays.toString(outputColumns);
   }
 
+  public List<String> outputColumnsAndTypesToStringList(int[] outputColumns, TypeInfo[] typeInfos) {
+    final int size = outputColumns.length;
+    ArrayList<String> result = new ArrayList<String>(size);
+    for (int i = 0; i < size; i++) {
+      result.add(outputColumns[i] + ":" + typeInfos[i].toString());
+    }
+    return result;
+  }
+
+  public List<String> outputColumnsAndTypesToStringList(VectorColumnMapping vectorColumnMapping) {
+    final int size = vectorColumnMapping.getCount();
+    if (size == 0) {
+      return null;
+    }
+    int[] outputColumns = vectorColumnMapping.getOutputColumns();
+    TypeInfo[] typeInfos = vectorColumnMapping.getTypeInfos();
+    return outputColumnsAndTypesToStringList(outputColumns, typeInfos);
+  }
+
   public List<String> columnMappingToStringList(VectorColumnMapping vectorColumnMapping) {
     final int size = vectorColumnMapping.getCount();
     if (size == 0) {

http://git-wip-us.apache.org/repos/asf/hive/blob/a37827ec/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceSinkDesc.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceSinkDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceSinkDesc.java
index 61216bc..b1d8e1f 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceSinkDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/ReduceSinkDesc.java
@@ -29,6 +29,7 @@ import java.util.Set;
 import org.apache.hadoop.hive.conf.HiveConf;
 import org.apache.hadoop.hive.ql.io.AcidUtils;
 import org.apache.hadoop.hive.ql.optimizer.signature.Signature;
+import org.apache.hadoop.hive.ql.exec.Utilities;
 import org.apache.hadoop.hive.ql.plan.Explain.Level;
 import org.apache.hadoop.hive.ql.plan.Explain.Vectorization;
 import org.apache.hadoop.hive.ql.plan.VectorReduceSinkDesc.ReduceSinkKeyType;
@@ -193,6 +194,17 @@ public class ReduceSinkDesc extends AbstractOperatorDesc {
     return outputKeyColumnNames;
   }
 
+  // NOTE: Debugging only.
+  @Explain(displayName = "output key column names", explainLevels = { Level.DEBUG })
+  public List<String> getOutputKeyColumnNamesDisplay() {
+    List<String> result = new ArrayList<String>();
+    for (String name : outputKeyColumnNames) {
+      result.add(Utilities.ReduceField.KEY.name() + "." + name);
+    }
+    return result;
+  }
+
+
   public void setOutputKeyColumnNames(
       java.util.ArrayList<java.lang.String> outputKeyColumnNames) {
     this.outputKeyColumnNames = outputKeyColumnNames;
@@ -202,6 +214,16 @@ public class ReduceSinkDesc extends AbstractOperatorDesc {
     return outputValueColumnNames;
   }
 
+  // NOTE: Debugging only.
+  @Explain(displayName = "output value column names", explainLevels = { Level.DEBUG })
+  public List<String> getOutputValueColumnNamesDisplay() {
+    List<String> result = new ArrayList<String>();
+    for (String name : outputValueColumnNames) {
+      result.add(Utilities.ReduceField.VALUE.name() + "." + name);
+    }
+    return result;
+  }
+
   public void setOutputValueColumnNames(
       java.util.ArrayList<java.lang.String> outputValueColumnNames) {
     this.outputValueColumnNames = outputValueColumnNames;
@@ -540,34 +562,41 @@ public class ReduceSinkDesc extends AbstractOperatorDesc {
       return vectorExpressionsToStringList(vectorReduceSinkInfo.getReduceSinkValueExpressions());
     }
 
-    @Explain(vectorization = Vectorization.DETAIL, displayName = "keyColumnNums", explainLevels = { Level.DEFAULT, Level.EXTENDED })
-    public String getKeyColumnNums() {
+    @Explain(vectorization = Vectorization.DETAIL, displayName = "keyColumns",
+        explainLevels = { Level.DEFAULT, Level.EXTENDED })
+    public List<String> getKeyColumns() {
       if (!isNative) {
         return null;
       }
       int[] keyColumnMap = vectorReduceSinkInfo.getReduceSinkKeyColumnMap();
       if (keyColumnMap == null) {
         // Always show an array.
-        keyColumnMap = new int[0];
+        return new ArrayList<String>();
       }
-      return Arrays.toString(keyColumnMap);
+      return outputColumnsAndTypesToStringList(
+          vectorReduceSinkInfo.getReduceSinkKeyColumnMap(),
+          vectorReduceSinkInfo.getReduceSinkKeyTypeInfos());
     }
 
-    @Explain(vectorization = Vectorization.DETAIL, displayName = "valueColumnNums", explainLevels = { Level.DEFAULT, Level.EXTENDED })
-    public String getValueColumnNums() {
+    @Explain(vectorization = Vectorization.DETAIL, displayName = "valueColumns",
+        explainLevels = { Level.DEFAULT, Level.EXTENDED })
+    public List<String> getValueColumns() {
       if (!isNative) {
         return null;
       }
       int[] valueColumnMap = vectorReduceSinkInfo.getReduceSinkValueColumnMap();
       if (valueColumnMap == null) {
         // Always show an array.
-        valueColumnMap = new int[0];
+        return new ArrayList<String>();
       }
-      return Arrays.toString(valueColumnMap);
+      return outputColumnsAndTypesToStringList(
+          vectorReduceSinkInfo.getReduceSinkValueColumnMap(),
+          vectorReduceSinkInfo.getReduceSinkValueTypeInfos());
     }
 
-    @Explain(vectorization = Vectorization.DETAIL, displayName = "bucketColumnNums", explainLevels = { Level.DEFAULT, Level.EXTENDED })
-    public String getBucketColumnNums() {
+    @Explain(vectorization = Vectorization.DETAIL, displayName = "bucketColumns",
+        explainLevels = { Level.DEFAULT, Level.EXTENDED })
+    public List<String> getBucketColumns() {
       if (!isNative) {
         return null;
       }
@@ -576,11 +605,14 @@ public class ReduceSinkDesc extends AbstractOperatorDesc {
         // Suppress empty column map.
         return null;
       }
-      return Arrays.toString(bucketColumnMap);
+      return outputColumnsAndTypesToStringList(
+          vectorReduceSinkInfo.getReduceSinkBucketColumnMap(),
+          vectorReduceSinkInfo.getReduceSinkBucketTypeInfos());
     }
 
-    @Explain(vectorization = Vectorization.DETAIL, displayName = "partitionColumnNums", explainLevels = { Level.DEFAULT, Level.EXTENDED })
-    public String getPartitionColumnNums() {
+    @Explain(vectorization = Vectorization.DETAIL, displayName = "partitionColumns",
+        explainLevels = { Level.DEFAULT, Level.EXTENDED })
+    public List<String> getPartitionColumns() {
       if (!isNative) {
         return null;
       }
@@ -589,7 +621,9 @@ public class ReduceSinkDesc extends AbstractOperatorDesc {
        // Suppress empty column map.
         return null;
       }
-      return Arrays.toString(partitionColumnMap);
+      return outputColumnsAndTypesToStringList(
+          vectorReduceSinkInfo.getReduceSinkPartitionColumnMap(),
+          vectorReduceSinkInfo.getReduceSinkPartitionTypeInfos());
     }
 
     private VectorizationCondition[] createNativeConditions() {
@@ -598,7 +632,8 @@ public class ReduceSinkDesc extends AbstractOperatorDesc {
 
       String engine = vectorReduceSinkDesc.getEngine();
       String engineInSupportedCondName =
-          HiveConf.ConfVars.HIVE_EXECUTION_ENGINE.varname + " " + engine + " IN " + vectorizableReduceSinkNativeEngines;
+          HiveConf.ConfVars.HIVE_EXECUTION_ENGINE.varname + " " + engine + " IN " +
+              vectorizableReduceSinkNativeEngines;
       boolean engineInSupported = vectorizableReduceSinkNativeEngines.contains(engine);
 
       VectorizationCondition[] conditions = new VectorizationCondition[] {
@@ -633,7 +668,8 @@ public class ReduceSinkDesc extends AbstractOperatorDesc {
       return conditions;
     }
 
-    @Explain(vectorization = Vectorization.OPERATOR, displayName = "nativeConditionsMet", explainLevels = { Level.DEFAULT, Level.EXTENDED })
+    @Explain(vectorization = Vectorization.OPERATOR, displayName = "nativeConditionsMet",
+        explainLevels = { Level.DEFAULT, Level.EXTENDED })
     public List<String> getNativeConditionsMet() {
       if (nativeConditions == null) {
         nativeConditions = createNativeConditions();
@@ -641,7 +677,8 @@ public class ReduceSinkDesc extends AbstractOperatorDesc {
       return VectorizationCondition.getConditionsMet(nativeConditions);
     }
 
-    @Explain(vectorization = Vectorization.OPERATOR, displayName = "nativeConditionsNotMet", explainLevels = { Level.DEFAULT, Level.EXTENDED })
+    @Explain(vectorization = Vectorization.OPERATOR, displayName = "nativeConditionsNotMet",
+        explainLevels = { Level.DEFAULT, Level.EXTENDED })
     public List<String> getNativeConditionsNotMet() {
       if (nativeConditions == null) {
         nativeConditions = createNativeConditions();
@@ -650,7 +687,8 @@ public class ReduceSinkDesc extends AbstractOperatorDesc {
     }
   }
 
-  @Explain(vectorization = Vectorization.OPERATOR, displayName = "Reduce Sink Vectorization", explainLevels = { Level.DEFAULT, Level.EXTENDED })
+  @Explain(vectorization = Vectorization.OPERATOR, displayName = "Reduce Sink Vectorization",
+      explainLevels = { Level.DEFAULT, Level.EXTENDED })
   public ReduceSinkOperatorExplainVectorization getReduceSinkVectorization() {
     VectorReduceSinkDesc vectorReduceSinkDesc = (VectorReduceSinkDesc) getVectorDesc();
     if (vectorReduceSinkDesc == null) {

http://git-wip-us.apache.org/repos/asf/hive/blob/a37827ec/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorMapJoinDesc.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorMapJoinDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorMapJoinDesc.java
index 58032ca..89a07b4 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorMapJoinDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorMapJoinDesc.java
@@ -85,11 +85,11 @@ public class VectorMapJoinDesc extends AbstractVectorDesc  {
   }
 
   public static enum VectorMapJoinVariation {
-    NONE,
-    INNER_BIG_ONLY,
     INNER,
+    INNER_BIG_ONLY,
     LEFT_SEMI,
-    OUTER
+    OUTER,
+    FULL_OUTER
   }
 
   private HashTableImplementationType hashTableImplementationType;
@@ -107,7 +107,7 @@ public class VectorMapJoinDesc extends AbstractVectorDesc  {
     hashTableImplementationType = HashTableImplementationType.NONE;
     hashTableKind = HashTableKind.NONE;
     hashTableKeyType = HashTableKeyType.NONE;
-    vectorMapJoinVariation = VectorMapJoinVariation.NONE;
+    vectorMapJoinVariation = null;
     minMaxEnabled = false;
 
     allBigTableKeyExpressions = null;
@@ -206,6 +206,7 @@ public class VectorMapJoinDesc extends AbstractVectorDesc  {
   private List<String> notSupportedKeyTypes;
   private boolean smallTableExprVectorizes;
   private boolean outerJoinHasNoKeys;
+  boolean isFullOuter;
 
   public void setUseOptimizedTable(boolean useOptimizedTable) {
     this.useOptimizedTable = useOptimizedTable;
@@ -274,5 +275,10 @@ public class VectorMapJoinDesc extends AbstractVectorDesc  {
   public boolean getIsHybridHashJoin() {
     return isHybridHashJoin;
   }
-
+  public void setIsFullOuter(boolean isFullOuter) {
+    this.isFullOuter = isFullOuter;
+  }
+  public boolean getIsFullOuter() {
+    return isFullOuter;
+  }
 }

http://git-wip-us.apache.org/repos/asf/hive/blob/a37827ec/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorMapJoinInfo.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorMapJoinInfo.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorMapJoinInfo.java
index 6db0540..ad82e5c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorMapJoinInfo.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorMapJoinInfo.java
@@ -48,9 +48,19 @@ public class VectorMapJoinInfo {
   private TypeInfo[] bigTableValueTypeInfos;
   private VectorExpression[] slimmedBigTableValueExpressions;
 
-  private VectorColumnOutputMapping bigTableRetainedMapping;
-  private VectorColumnOutputMapping bigTableOuterKeyMapping;
-  private VectorColumnSourceMapping smallTableMapping;
+  private VectorExpression[] bigTableFilterExpressions;
+
+  private int[] bigTableRetainColumnMap;
+  private TypeInfo[] bigTableRetainTypeInfos;
+
+  private int[] nonOuterSmallTableKeyColumnMap;
+  private TypeInfo[] nonOuterSmallTableKeyTypeInfos;
+
+  private VectorColumnOutputMapping outerSmallTableKeyMapping;
+
+  private VectorColumnSourceMapping fullOuterSmallTableKeyMapping;
+
+  private VectorColumnSourceMapping smallTableValueMapping;
 
   private VectorColumnSourceMapping projectionMapping;
 
@@ -65,9 +75,19 @@ public class VectorMapJoinInfo {
     bigTableValueTypeInfos = null;
     slimmedBigTableValueExpressions = null;
 
-    bigTableRetainedMapping = null;
-    bigTableOuterKeyMapping = null;
-    smallTableMapping = null;
+    bigTableFilterExpressions = null;
+
+    bigTableRetainColumnMap = null;
+    bigTableRetainTypeInfos = null;
+
+    nonOuterSmallTableKeyColumnMap = null;
+    nonOuterSmallTableKeyTypeInfos = null;
+
+    outerSmallTableKeyMapping = null;
+
+    fullOuterSmallTableKeyMapping = null;
+
+    smallTableValueMapping = null;
 
     projectionMapping = null;
   }
@@ -138,28 +158,69 @@ public class VectorMapJoinInfo {
     this.slimmedBigTableValueExpressions = slimmedBigTableValueExpressions;
   }
 
-  public void setBigTableRetainedMapping(VectorColumnOutputMapping bigTableRetainedMapping) {
-    this.bigTableRetainedMapping = bigTableRetainedMapping;
+  public VectorExpression[] getBigTableFilterExpressions() {
+    return bigTableFilterExpressions;
+  }
+
+  public void setBigTableFilterExpressions(VectorExpression[] bigTableFilterExpressions) {
+    this.bigTableFilterExpressions = bigTableFilterExpressions;
+  }
+
+  public void setBigTableRetainColumnMap(int[] bigTableRetainColumnMap) {
+    this.bigTableRetainColumnMap = bigTableRetainColumnMap;
+  }
+
+  public int[] getBigTableRetainColumnMap() {
+    return bigTableRetainColumnMap;
+  }
+
+  public void setBigTableRetainTypeInfos(TypeInfo[] bigTableRetainTypeInfos) {
+    this.bigTableRetainTypeInfos = bigTableRetainTypeInfos;
+  }
+
+  public TypeInfo[] getBigTableRetainTypeInfos() {
+    return bigTableRetainTypeInfos;
+  }
+
+  public void setNonOuterSmallTableKeyColumnMap(int[] nonOuterSmallTableKeyColumnMap) {
+    this.nonOuterSmallTableKeyColumnMap = nonOuterSmallTableKeyColumnMap;
+  }
+
+  public int[] getNonOuterSmallTableKeyColumnMap() {
+    return nonOuterSmallTableKeyColumnMap;
+  }
+
+  public void setNonOuterSmallTableKeyTypeInfos(TypeInfo[] nonOuterSmallTableKeyTypeInfos) {
+    this.nonOuterSmallTableKeyTypeInfos = nonOuterSmallTableKeyTypeInfos;
+  }
+
+  public TypeInfo[] getNonOuterSmallTableKeyTypeInfos() {
+    return nonOuterSmallTableKeyTypeInfos;
+  }
+
+  public void setOuterSmallTableKeyMapping(VectorColumnOutputMapping outerSmallTableKeyMapping) {
+    this.outerSmallTableKeyMapping = outerSmallTableKeyMapping;
   }
 
-  public VectorColumnOutputMapping getBigTableRetainedMapping() {
-    return bigTableRetainedMapping;
+  public VectorColumnOutputMapping getOuterSmallTableKeyMapping() {
+    return outerSmallTableKeyMapping;
   }
 
-  public void setBigTableOuterKeyMapping(VectorColumnOutputMapping bigTableOuterKeyMapping) {
-    this.bigTableOuterKeyMapping = bigTableOuterKeyMapping;
+  public void setFullOuterSmallTableKeyMapping(
+      VectorColumnSourceMapping fullOuterSmallTableKeyMapping) {
+    this.fullOuterSmallTableKeyMapping = fullOuterSmallTableKeyMapping;
   }
 
-  public VectorColumnOutputMapping getBigTableOuterKeyMapping() {
-    return bigTableOuterKeyMapping;
+  public VectorColumnSourceMapping getFullOuterSmallTableKeyMapping() {
+    return fullOuterSmallTableKeyMapping;
   }
 
-  public void setSmallTableMapping(VectorColumnSourceMapping smallTableMapping) {
-    this.smallTableMapping = smallTableMapping;
+  public void setSmallTableValueMapping(VectorColumnSourceMapping smallTableValueMapping) {
+    this.smallTableValueMapping = smallTableValueMapping;
   }
 
-  public VectorColumnSourceMapping getSmallTableMapping() {
-    return smallTableMapping;
+  public VectorColumnSourceMapping getSmallTableValueMapping() {
+    return smallTableValueMapping;
   }
 
   public void setProjectionMapping(VectorColumnSourceMapping projectionMapping) {

http://git-wip-us.apache.org/repos/asf/hive/blob/a37827ec/ql/src/test/org/apache/hadoop/hive/ql/exec/persistence/TestBytesBytesMultiHashMap.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/persistence/TestBytesBytesMultiHashMap.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/persistence/TestBytesBytesMultiHashMap.java
index 9f785e6..e5c749f 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/exec/persistence/TestBytesBytesMultiHashMap.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/persistence/TestBytesBytesMultiHashMap.java
@@ -84,9 +84,9 @@ public class TestBytesBytesMultiHashMap {
     map.put(kv2, -1);
     key[0] = (byte)(key[0] + 1);
     BytesBytesMultiHashMap.Result hashMapResult = new BytesBytesMultiHashMap.Result();
-    map.getValueResult(key, 0, key.length, hashMapResult);
+    map.getValueResult(key, 0, key.length, hashMapResult, null);
     assertTrue(!hashMapResult.hasRows());
-    map.getValueResult(key, 0, 0, hashMapResult);
+    map.getValueResult(key, 0, 0, hashMapResult, null);
     assertTrue(!hashMapResult.hasRows());
   }
 
@@ -104,7 +104,7 @@ public class TestBytesBytesMultiHashMap {
     assertEquals(CAPACITY, map.getCapacity());
     // Get of non-existent key should terminate..
     BytesBytesMultiHashMap.Result hashMapResult = new BytesBytesMultiHashMap.Result();
-    map.getValueResult(new byte[0], 0, 0, hashMapResult);
+    map.getValueResult(new byte[0], 0, 0, hashMapResult, null);
   }
 
   @Test
@@ -123,7 +123,7 @@ public class TestBytesBytesMultiHashMap {
 
   private void verifyHashMapResult(BytesBytesMultiHashMap map, byte[] key, byte[]... values) {
     BytesBytesMultiHashMap.Result hashMapResult = new BytesBytesMultiHashMap.Result();
-    byte state = map.getValueResult(key, 0, key.length, hashMapResult);
+    byte state = map.getValueResult(key, 0, key.length, hashMapResult, null);
     HashSet<ByteBuffer> hs = new HashSet<ByteBuffer>();
     int count = 0;
     if (hashMapResult.hasRows()) {

http://git-wip-us.apache.org/repos/asf/hive/blob/a37827ec/ql/src/test/org/apache/hadoop/hive/ql/exec/util/collectoroperator/CollectorTestOperator.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/util/collectoroperator/CollectorTestOperator.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/util/collectoroperator/CollectorTestOperator.java
index 6491d79..244208b 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/exec/util/collectoroperator/CollectorTestOperator.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/util/collectoroperator/CollectorTestOperator.java
@@ -26,8 +26,22 @@ public class CollectorTestOperator extends Operator {
 
   private static final long serialVersionUID = 1L;
 
+  private boolean isClosed;
+  private boolean isAborted;
+
   public CollectorTestOperator() {
     super();
+
+    isClosed = false;
+    isAborted = false;
+  }
+
+  public boolean getIsClosed() {
+    return isClosed;
+  }
+
+  public boolean getIsAborted() {
+    return isAborted;
   }
 
   @Override
@@ -36,6 +50,14 @@ public class CollectorTestOperator extends Operator {
   }
 
   @Override
+  public void closeOp(boolean abort) {
+    isClosed = true;
+    if (abort) {
+      isAborted = true;
+    }
+  }
+
+  @Override
   public String getName() {
     return CollectorTestOperator.class.getSimpleName();
   }