You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by jc...@apache.org on 2016/12/06 00:27:34 UTC
[7/7] hive git commit: HIVE-15251: Provide support for complex
expressions in ON clauses for OUTER joins (Jesus Camacho Rodriguez,
reviewed by Ashutosh Chauhan)
HIVE-15251: Provide support for complex expressions in ON clauses for OUTER joins (Jesus Camacho Rodriguez, reviewed by Ashutosh Chauhan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/9b2badc3
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/9b2badc3
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/9b2badc3
Branch: refs/heads/master
Commit: 9b2badc3e1c8fdb84a78bcbf503f19c219b4b5c0
Parents: 7089ac7
Author: Jesus Camacho Rodriguez <jc...@apache.org>
Authored: Fri Dec 2 17:15:53 2016 +0000
Committer: Jesus Camacho Rodriguez <jc...@apache.org>
Committed: Tue Dec 6 00:26:19 2016 +0000
----------------------------------------------------------------------
.../test/resources/testconfiguration.properties | 5 +-
.../org/apache/hadoop/hive/ql/ErrorMsg.java | 2 +-
.../hadoop/hive/ql/exec/CommonJoinOperator.java | 295 ++-
.../apache/hadoop/hive/ql/exec/JoinUtil.java | 17 +
.../hive/ql/optimizer/ConvertJoinMapJoin.java | 2 +
.../hive/ql/optimizer/MapJoinProcessor.java | 1 +
.../hive/ql/optimizer/physical/Vectorizer.java | 4 +
.../hadoop/hive/ql/parse/SemanticAnalyzer.java | 32 +-
.../apache/hadoop/hive/ql/plan/JoinDesc.java | 33 +
ql/src/test/queries/clientnegative/join45.q | 13 -
ql/src/test/queries/clientpositive/join46.q | 263 +++
ql/src/test/queries/clientpositive/mapjoin46.q | 264 +++
.../queries/clientpositive/smb_mapjoin_46.q | 240 ++
.../queries/clientpositive/vectorized_join46.q | 225 ++
ql/src/test/results/clientnegative/join45.q.out | 13 -
ql/src/test/results/clientpositive/join46.q.out | 1948 ++++++++++++++++
.../results/clientpositive/llap/join46.q.out | 2190 ++++++++++++++++++
.../results/clientpositive/llap/mapjoin46.q.out | 2101 +++++++++++++++++
.../clientpositive/llap/vectorized_join46.q.out | 1867 +++++++++++++++
.../test/results/clientpositive/mapjoin46.q.out | 2002 ++++++++++++++++
.../results/clientpositive/smb_mapjoin_46.q.out | 1695 ++++++++++++++
21 files changed, 13111 insertions(+), 101 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/9b2badc3/itests/src/test/resources/testconfiguration.properties
----------------------------------------------------------------------
diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties
index 772e123..5db98f1 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -181,6 +181,7 @@ minillaplocal.shared.query.files=alter_merge_2_orc.q,\
join0.q,\
join1.q,\
join_emit_interval.q,\
+ join46.q,\
join_nullsafe.q,\
leftsemijoin.q,\
limit_pushdown.q,\
@@ -191,6 +192,7 @@ minillaplocal.shared.query.files=alter_merge_2_orc.q,\
mapjoin2.q,\
mapjoin_decimal.q,\
mapjoin_mapjoin.q,\
+ mapjoin46.q,\
merge1.q,\
merge2.q,\
mergejoin.q,\
@@ -690,7 +692,8 @@ minillaplocal.query.files=acid_globallimit.q,\
database.q,\
smb_mapjoin_17.q,\
groupby_resolution.q,\
- windowing_windowspec2.q
+ windowing_windowspec2.q,\
+ vectorized_join46.q
encrypted.query.files=encryption_join_unencrypted_tbl.q,\
encryption_insert_partition_static.q,\
http://git-wip-us.apache.org/repos/asf/hive/blob/9b2badc3/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java b/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
index b62df35..a315057 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
@@ -459,7 +459,7 @@ public enum ErrorMsg {
"requires \"AND <boolean>\" on the 1st WHEN MATCHED clause of <{0}>", true),
MERGE_TOO_MANY_DELETE(10405, "MERGE statment can have at most 1 WHEN MATCHED ... DELETE clause: <{0}>", true),
MERGE_TOO_MANY_UPDATE(10406, "MERGE statment can have at most 1 WHEN MATCHED ... UPDATE clause: <{0}>", true),
- INVALID_JOIN_CONDITION(10407, "Complex condition not supported for (LEFT|RIGHT|FULL) OUTER JOIN"),
+ INVALID_JOIN_CONDITION(10407, "Error parsing condition in outer join"),
//========================== 20000 range starts here ========================//
SCRIPT_INIT_ERROR(20000, "Unable to initialize custom script."),
SCRIPT_IO_ERROR(20001, "An error occurred while reading or writing to your custom script. "
http://git-wip-us.apache.org/repos/asf/hive/blob/9b2badc3/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java
index 5512ee2..940f2dd 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.hive.ql.exec;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -31,6 +32,7 @@ import org.apache.hadoop.hive.ql.CompilationOpContext;
import org.apache.hadoop.hive.ql.exec.persistence.AbstractRowContainer;
import org.apache.hadoop.hive.ql.exec.persistence.RowContainer;
import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
import org.apache.hadoop.hive.ql.plan.JoinCondDesc;
import org.apache.hadoop.hive.ql.plan.JoinDesc;
import org.apache.hadoop.hive.ql.plan.TableDesc;
@@ -63,6 +65,16 @@ public abstract class CommonJoinOperator<T extends JoinDesc> extends
*/
protected transient List<ExprNodeEvaluator>[] joinFilters;
+ /**
+ * List of evaluators for conditions which appear on on-clause and needs to be
+ * evaluated before emitting rows. Currently, relevant only for outer joins.
+ *
+ * For instance, given the query:
+ * select * from t1 right outer join t2 on t1.c1 + t2.c2 > t1.c3;
+ * The expression evaluator for t1.c1 + t2.c2 > t1.c3 will be stored in this list.
+ */
+ protected transient List<ExprNodeEvaluator> residualJoinFilters;
+
protected transient int[][] filterMaps;
/**
@@ -74,6 +86,24 @@ public abstract class CommonJoinOperator<T extends JoinDesc> extends
* The ObjectInspectors for join filters.
*/
protected transient List<ObjectInspector>[] joinFilterObjectInspectors;
+
+ /**
+ * OIs corresponding to residualJoinFilters.
+ */
+ protected transient List<ObjectInspector> residualJoinFiltersOIs;
+
+ /**
+ * Will be true depending on content of residualJoinFilters.
+ */
+ protected transient boolean needsPostEvaluation;
+
+ /**
+ * This data structure is used to keep track of rows on which residualFilters
+ * evaluated to false. We will iterate on this container afterwards and emit
+ * rows appending NULL values if it was not done. Key is relation index.
+ */
+ protected transient Map<Integer, Object[]> rowContainerPostFilteredOuterJoin = null;
+
/**
* The standard ObjectInspectors for the join inputs.
*/
@@ -164,6 +194,9 @@ public abstract class CommonJoinOperator<T extends JoinDesc> extends
this.statsMap = clone.statsMap;
this.joinFilters = clone.joinFilters;
this.joinFilterObjectInspectors = clone.joinFilterObjectInspectors;
+ this.residualJoinFilters = clone.residualJoinFilters;
+ this.residualJoinFiltersOIs = clone.residualJoinFiltersOIs;
+ this.needsPostEvaluation = clone.needsPostEvaluation;
}
private <T extends JoinDesc> ObjectInspector getJoinOutputObjectInspector(
@@ -247,7 +280,7 @@ public abstract class CommonJoinOperator<T extends JoinDesc> extends
rowContainerObjectInspectors[alias] = rcOIs;
}
rowContainerStandardObjectInspectors =
- JoinUtil.getStandardObjectInspectors(rowContainerObjectInspectors,NOTSKIPBIGTABLE, tagLen);
+ JoinUtil.getStandardObjectInspectors(rowContainerObjectInspectors, NOTSKIPBIGTABLE, tagLen);
}
dummyObj = new ArrayList[numAliases];
@@ -323,6 +356,30 @@ public abstract class CommonJoinOperator<T extends JoinDesc> extends
}
}
+ // Create post-filtering evaluators if needed
+ if (conf.getResidualFilterExprs() != null) {
+ // Currently residual filter expressions are only used with outer joins, thus
+ // we add this safeguard.
+ // TODO: Remove this guard when support for residual expressions can be present
+ // for inner joins too. This would be added to improve efficiency in the evaluation
+ // of certain joins, since we will not be emitting rows which are thrown away by
+ // filter straight away.
+ assert !noOuterJoin;
+ residualJoinFilters = new ArrayList<>(conf.getResidualFilterExprs().size());
+ residualJoinFiltersOIs = new ArrayList<>(conf.getResidualFilterExprs().size());
+ for (int i = 0; i < conf.getResidualFilterExprs().size(); i++) {
+ ExprNodeDesc expr = conf.getResidualFilterExprs().get(i);
+ residualJoinFilters.add(ExprNodeEvaluatorFactory.get(expr));
+ residualJoinFiltersOIs.add(
+ residualJoinFilters.get(i).initialize(outputObjInspector));
+ }
+ needsPostEvaluation = true;
+ // We need to disable join emit interval, since for outer joins with post conditions
+ // we need to have the full view on the right matching rows to know whether we need
+ // to produce a row with NULL values or not
+ joinEmitInterval = -1;
+ }
+
if (isLogInfoEnabled) {
LOG.info("JOIN " + outputObjInspector.getTypeName() + " totalsz = " + totalSz);
}
@@ -426,7 +483,8 @@ public abstract class CommonJoinOperator<T extends JoinDesc> extends
}
// fill forwardCache with skipvector
- private void createForwardJoinObject(boolean[] skip) throws HiveException {
+ // returns whether a record was forwarded
+ private boolean createForwardJoinObject(boolean[] skip) throws HiveException {
Arrays.fill(forwardCache, null);
boolean forward = false;
@@ -439,13 +497,30 @@ public abstract class CommonJoinOperator<T extends JoinDesc> extends
}
}
if (forward) {
- internalForward(forwardCache, outputObjInspector);
- countAfterReport = 0;
+ if (needsPostEvaluation) {
+ forward = !JoinUtil.isFiltered(forwardCache, residualJoinFilters, residualJoinFiltersOIs);
+ }
+ if (forward) {
+ // If it is not an outer join, or the post-condition filters
+ // are empty or the row passed them
+ internalForward(forwardCache, outputObjInspector);
+ countAfterReport = 0;
+ }
}
+
+ return forward;
}
// entry point (aliasNum = 0)
private void genJoinObject() throws HiveException {
+ if (needsPostEvaluation && 0 == numAliases - 2) {
+ int nextType = condn[0].getType();
+ if (nextType == JoinDesc.RIGHT_OUTER_JOIN || nextType == JoinDesc.FULL_OUTER_JOIN) {
+ // Initialize container to use for storing tuples before emitting them
+ rowContainerPostFilteredOuterJoin = new HashMap<>();
+ }
+ }
+
boolean rightFirst = true;
boolean hasFilter = hasFilter(order[0]);
AbstractRowContainer.RowIterator<List<Object>> iter = storage[order[0]].rowIter();
@@ -460,78 +535,173 @@ public abstract class CommonJoinOperator<T extends JoinDesc> extends
genObject(1, rightFirst, rightNull);
rightFirst = false;
}
+
+ // Consolidation for outer joins
+ if (needsPostEvaluation && 0 == numAliases - 2) {
+ int nextType = condn[0].getType();
+ if (nextType == JoinDesc.RIGHT_OUTER_JOIN || nextType == JoinDesc.FULL_OUTER_JOIN) {
+ // If it is a RIGHT / FULL OUTER JOIN, we need to iterate through the row container
+ // that contains all the right records that did not produce results. Then, for each
+ // of those records, we replace the left side with NULL values, and produce the
+ // records.
+ // Observe that we only enter this block when we have finished iterating through
+ // all the left and right records (aliasNum == numAliases - 2), and thus, we have
+ // tried to evaluate the post-filter condition on every possible combination.
+ // NOTE: the left records that do not produce results (for LEFT / FULL OUTER JOIN)
+ // will always be caught in the genObject method
+ Arrays.fill(forwardCache, null);
+ for (Object[] row : rowContainerPostFilteredOuterJoin.values()) {
+ if (row == null) {
+ continue;
+ }
+ System.arraycopy(row, 0, forwardCache, offsets[numAliases - 1], row.length);
+ internalForward(forwardCache, outputObjInspector);
+ countAfterReport = 0;
+ }
+ }
+ }
}
// creates objects in recursive manner
private void genObject(int aliasNum, boolean allLeftFirst, boolean allLeftNull)
throws HiveException {
- if (aliasNum < numAliases) {
+ JoinCondDesc joinCond = condn[aliasNum - 1];
+ int type = joinCond.getType();
+ int left = joinCond.getLeft();
+ int right = joinCond.getRight();
+
+ if (needsPostEvaluation && aliasNum == numAliases - 2) {
+ int nextType = condn[aliasNum].getType();
+ if (nextType == JoinDesc.RIGHT_OUTER_JOIN || nextType == JoinDesc.FULL_OUTER_JOIN) {
+ // Initialize container to use for storing tuples before emitting them
+ rowContainerPostFilteredOuterJoin = new HashMap<>();
+ }
+ }
- boolean[] skip = skipVectors[aliasNum];
- boolean[] prevSkip = skipVectors[aliasNum - 1];
+ boolean[] skip = skipVectors[aliasNum];
+ boolean[] prevSkip = skipVectors[aliasNum - 1];
- JoinCondDesc joinCond = condn[aliasNum - 1];
- int type = joinCond.getType();
- int left = joinCond.getLeft();
- int right = joinCond.getRight();
+ // search for match in the rhs table
+ AbstractRowContainer<List<Object>> aliasRes = storage[order[aliasNum]];
- // search for match in the rhs table
- AbstractRowContainer<List<Object>> aliasRes = storage[order[aliasNum]];
+ boolean needToProduceLeftRow = false;
+ boolean producedRow = false;
+ boolean done = false;
+ boolean loopAgain = false;
+ boolean tryLOForFO = type == JoinDesc.FULL_OUTER_JOIN;
- boolean done = false;
- boolean loopAgain = false;
- boolean tryLOForFO = type == JoinDesc.FULL_OUTER_JOIN;
+ boolean rightFirst = true;
+ AbstractRowContainer.RowIterator<List<Object>> iter = aliasRes.rowIter();
+ int pos = 0;
+ for (List<Object> rightObj = iter.first(); !done && rightObj != null;
+ rightObj = loopAgain ? rightObj : iter.next(), rightFirst = loopAgain = false, pos++) {
+ System.arraycopy(prevSkip, 0, skip, 0, prevSkip.length);
+
+ boolean rightNull = rightObj == dummyObj[aliasNum];
+ if (hasFilter(order[aliasNum])) {
+ filterTags[aliasNum] = getFilterTag(rightObj);
+ }
+ skip[right] = rightNull;
+
+ if (type == JoinDesc.INNER_JOIN) {
+ innerJoin(skip, left, right);
+ } else if (type == JoinDesc.LEFT_SEMI_JOIN) {
+ if (innerJoin(skip, left, right)) {
+ // if left-semi-join found a match, skipping the rest of the rows in the
+ // rhs table of the semijoin
+ done = true;
+ }
+ } else if (type == JoinDesc.LEFT_OUTER_JOIN ||
+ (type == JoinDesc.FULL_OUTER_JOIN && rightNull)) {
+ int result = leftOuterJoin(skip, left, right);
+ if (result < 0) {
+ continue;
+ }
+ done = result > 0;
+ } else if (type == JoinDesc.RIGHT_OUTER_JOIN ||
+ (type == JoinDesc.FULL_OUTER_JOIN && allLeftNull)) {
+ if (allLeftFirst && !rightOuterJoin(skip, left, right) ||
+ !allLeftFirst && !innerJoin(skip, left, right)) {
+ continue;
+ }
+ } else if (type == JoinDesc.FULL_OUTER_JOIN) {
+ if (tryLOForFO && leftOuterJoin(skip, left, right) > 0) {
+ loopAgain = allLeftFirst;
+ done = !loopAgain;
+ tryLOForFO = false;
+ } else if (allLeftFirst && !rightOuterJoin(skip, left, right) ||
+ !allLeftFirst && !innerJoin(skip, left, right)) {
+ continue;
+ }
+ }
+ intermediate[aliasNum] = rightObj;
- boolean rightFirst = true;
- AbstractRowContainer.RowIterator<List<Object>> iter = aliasRes.rowIter();
- for (List<Object> rightObj = iter.first(); !done && rightObj != null;
- rightObj = loopAgain ? rightObj : iter.next(), rightFirst = loopAgain = false) {
- System.arraycopy(prevSkip, 0, skip, 0, prevSkip.length);
+ if (aliasNum == numAliases - 1) {
+ if (!(allLeftNull && rightNull)) {
+ needToProduceLeftRow = true;
+ if (needsPostEvaluation) {
+ // This is only executed for outer joins with residual filters
+ boolean forward = createForwardJoinObject(skipVectors[numAliases - 1]);
+ producedRow |= forward;
+ if (!rightNull &&
+ (type == JoinDesc.RIGHT_OUTER_JOIN || type == JoinDesc.FULL_OUTER_JOIN)) {
+ if (forward) {
+ // This record produced a result this time, remove it from the storage
+ // as it will not need to produce a result with NULL values anymore
+ rowContainerPostFilteredOuterJoin.put(pos, null);
+ } else {
+ // We need to store this record (if it is not done yet) in case
+ // we should produce a result
+ if (!rowContainerPostFilteredOuterJoin.containsKey(pos)) {
+ Object[] row = Arrays.copyOfRange(forwardCache, offsets[aliasNum], offsets[aliasNum + 1]);
+ rowContainerPostFilteredOuterJoin.put(pos, row);
+ }
+ }
+ }
+ } else {
+ createForwardJoinObject(skipVectors[numAliases - 1]);
+ }
+ }
+ } else {
+ // recursively call the join the other rhs tables
+ genObject(aliasNum + 1, allLeftFirst && rightFirst, allLeftNull && rightNull);
+ }
+ }
- boolean rightNull = rightObj == dummyObj[aliasNum];
- if (hasFilter(order[aliasNum])) {
- filterTags[aliasNum] = getFilterTag(rightObj);
+ // Consolidation for outer joins
+ if (needsPostEvaluation && aliasNum == numAliases - 1 &&
+ needToProduceLeftRow && !producedRow && !allLeftNull) {
+ if (type == JoinDesc.LEFT_OUTER_JOIN || type == JoinDesc.FULL_OUTER_JOIN) {
+ // If it is a LEFT / FULL OUTER JOIN and the left record did not produce
+ // results, we need to take that record, replace the right side with NULL
+ // values, and produce the records
+ int i = numAliases - 1;
+ for (int j = offsets[i]; j < offsets[i + 1]; j++) {
+ forwardCache[j] = null;
}
- skip[right] = rightNull;
-
- if (type == JoinDesc.INNER_JOIN) {
- innerJoin(skip, left, right);
- } else if (type == JoinDesc.LEFT_SEMI_JOIN) {
- if (innerJoin(skip, left, right)) {
- // if left-semi-join found a match, skipping the rest of the rows in the
- // rhs table of the semijoin
- done = true;
- }
- } else if (type == JoinDesc.LEFT_OUTER_JOIN ||
- (type == JoinDesc.FULL_OUTER_JOIN && rightNull)) {
- int result = leftOuterJoin(skip, left, right);
- if (result < 0) {
- continue;
- }
- done = result > 0;
- } else if (type == JoinDesc.RIGHT_OUTER_JOIN ||
- (type == JoinDesc.FULL_OUTER_JOIN && allLeftNull)) {
- if (allLeftFirst && !rightOuterJoin(skip, left, right) ||
- !allLeftFirst && !innerJoin(skip, left, right)) {
- continue;
- }
- } else if (type == JoinDesc.FULL_OUTER_JOIN) {
- if (tryLOForFO && leftOuterJoin(skip, left, right) > 0) {
- loopAgain = allLeftFirst;
- done = !loopAgain;
- tryLOForFO = false;
- } else if (allLeftFirst && !rightOuterJoin(skip, left, right) ||
- !allLeftFirst && !innerJoin(skip, left, right)) {
+ internalForward(forwardCache, outputObjInspector);
+ countAfterReport = 0;
+ }
+ } else if (needsPostEvaluation && aliasNum == numAliases - 2) {
+ int nextType = condn[aliasNum].getType();
+ if (nextType == JoinDesc.RIGHT_OUTER_JOIN || nextType == JoinDesc.FULL_OUTER_JOIN) {
+ // If it is a RIGHT / FULL OUTER JOIN, we need to iterate through the row container
+ // that contains all the right records that did not produce results. Then, for each
+ // of those records, we replace the left side with NULL values, and produce the
+ // records.
+ // Observe that we only enter this block when we have finished iterating through
+ // all the left and right records (aliasNum == numAliases - 2), and thus, we have
+ // tried to evaluate the post-filter condition on every possible combination.
+ Arrays.fill(forwardCache, null);
+ for (Object[] row : rowContainerPostFilteredOuterJoin.values()) {
+ if (row == null) {
continue;
}
+ System.arraycopy(row, 0, forwardCache, offsets[numAliases - 1], row.length);
+ internalForward(forwardCache, outputObjInspector);
+ countAfterReport = 0;
}
- intermediate[aliasNum] = rightObj;
-
- // recursively call the join the other rhs tables
- genObject(aliasNum + 1, allLeftFirst && rightFirst, allLeftNull && rightNull);
}
- } else if (!allLeftNull) {
- createForwardJoinObject(skipVectors[numAliases - 1]);
}
}
@@ -676,7 +846,6 @@ public abstract class CommonJoinOperator<T extends JoinDesc> extends
forwardCache[p++] = obj.get(j);
}
}
-
internalForward(forwardCache, outputObjInspector);
countAfterReport = 0;
}
@@ -754,9 +923,9 @@ public abstract class CommonJoinOperator<T extends JoinDesc> extends
}
}
- if (!hasEmpty && !mayHasMoreThanOne) {
+ if (!needsPostEvaluation && !hasEmpty && !mayHasMoreThanOne) {
genAllOneUniqueJoinObject();
- } else if (!hasEmpty && !hasLeftSemiJoin) {
+ } else if (!needsPostEvaluation && !hasEmpty && !hasLeftSemiJoin) {
genUniqueJoinObject(0, 0);
} else {
genJoinObject();
http://git-wip-us.apache.org/repos/asf/hive/blob/9b2badc3/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinUtil.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinUtil.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinUtil.java
index 6cbcab6..9718c48 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinUtil.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinUtil.java
@@ -233,6 +233,23 @@ public class JoinUtil {
/**
* Returns true if the row does not pass through filters.
*/
+ protected static boolean isFiltered(Object row, List<ExprNodeEvaluator> filters,
+ List<ObjectInspector> filtersOIs) throws HiveException {
+ for (int i = 0; i < filters.size(); i++) {
+ ExprNodeEvaluator evaluator = filters.get(i);
+ Object condition = evaluator.evaluate(row);
+ Boolean result = (Boolean) ((PrimitiveObjectInspector) filtersOIs.get(i)).
+ getPrimitiveJavaObject(condition);
+ if (result == null || !result) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Returns true if the row does not pass through filters.
+ */
protected static short isFiltered(Object row, List<ExprNodeEvaluator> filters,
List<ObjectInspector> ois, int[] filterMap) throws HiveException {
// apply join filters on the row.
http://git-wip-us.apache.org/repos/asf/hive/blob/9b2badc3/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java
index 7441f1e..beed6b8 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java
@@ -99,6 +99,7 @@ public class ConvertJoinMapJoin implements NodeProcessor {
return retval;
} else {
fallbackToReduceSideJoin(joinOp, context);
+ return null;
}
}
@@ -232,6 +233,7 @@ public class ConvertJoinMapJoin implements NodeProcessor {
joinDesc.getFilters(), joinDesc.getNoOuterJoin(), null);
mapJoinDesc.setNullSafes(joinDesc.getNullSafes());
mapJoinDesc.setFilterMap(joinDesc.getFilterMap());
+ mapJoinDesc.setResidualFilterExprs(joinDesc.getResidualFilterExprs());
mapJoinDesc.resetOrder();
}
http://git-wip-us.apache.org/repos/asf/hive/blob/9b2badc3/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java
index c6efd5b..b2893e7 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java
@@ -1192,6 +1192,7 @@ public class MapJoinProcessor extends Transform {
mapJoinDescriptor.setTagOrder(tagOrder);
mapJoinDescriptor.setNullSafes(desc.getNullSafes());
mapJoinDescriptor.setFilterMap(desc.getFilterMap());
+ mapJoinDescriptor.setResidualFilterExprs(desc.getResidualFilterExprs());
if (!valueIndices.isEmpty()) {
mapJoinDescriptor.setValueIndices(valueIndices);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/9b2badc3/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
index 37baaf6..108c4e6 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
@@ -1561,6 +1561,10 @@ public class Vectorizer implements PhysicalPlanResolver {
LOG.info("Cannot vectorize map work small table expression");
return false;
}
+ if (desc.getResidualFilterExprs() != null && !desc.getResidualFilterExprs().isEmpty()) {
+ LOG.info("Cannot vectorize outer join with complex ON clause");
+ return false;
+ }
return true;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/9b2badc3/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
index 42a7ab9..2d1118c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@ -7888,6 +7888,16 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
join.getNoOuterJoin(), joinCondns, filterMap, joinKeys);
desc.setReversedExprs(reversedExprs);
desc.setFilterMap(join.getFilterMap());
+ // For outer joins, add filters that apply to more than one input
+ if (!join.getNoOuterJoin() && join.getPostJoinFilters().size() != 0) {
+ List<ExprNodeDesc> residualFilterExprs = new ArrayList<ExprNodeDesc>();
+ for (ASTNode cond : join.getPostJoinFilters()) {
+ residualFilterExprs.add(genExprNodeDesc(cond, outputRR));
+ }
+ desc.setResidualFilterExprs(residualFilterExprs);
+ // Clean post-conditions
+ join.getPostJoinFilters().clear();
+ }
JoinOperator joinOp = (JoinOperator) OperatorFactory.getAndMakeChild(getOpContext(), desc,
new RowSchema(outputRR.getColumnInfos()), rightOps);
@@ -8102,18 +8112,20 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
joinOp.getConf().setQBJoinTreeProps(joinTree);
joinContext.put(joinOp, joinTree);
- // Safety check for postconditions; currently we do not support them for outer join
- if (joinTree.getPostJoinFilters().size() != 0 && !joinTree.getNoOuterJoin()) {
- throw new SemanticException(ErrorMsg.INVALID_JOIN_CONDITION.getMsg());
- }
- Operator op = joinOp;
- for(ASTNode condn : joinTree.getPostJoinFilters()) {
- op = genFilterPlan(qb, condn, op, false);
- if (LOG.isDebugEnabled()) {
- LOG.debug("Generated " + op + " with post-filtering conditions after JOIN operator");
+ if (joinTree.getPostJoinFilters().size() != 0) {
+ // Safety check for postconditions
+ assert joinTree.getNoOuterJoin();
+ Operator op = joinOp;
+ for(ASTNode condn : joinTree.getPostJoinFilters()) {
+ op = genFilterPlan(qb, condn, op, false);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Generated " + op + " with post-filtering conditions after JOIN operator");
+ }
}
+ return op;
}
- return op;
+
+ return joinOp;
}
/**
http://git-wip-us.apache.org/repos/asf/hive/blob/9b2badc3/ql/src/java/org/apache/hadoop/hive/ql/plan/JoinDesc.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/JoinDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/JoinDesc.java
index 2ca6b8f..3cd611c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/JoinDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/JoinDesc.java
@@ -59,6 +59,8 @@ public class JoinDesc extends AbstractOperatorDesc {
// alias to filter mapping
private Map<Byte, List<ExprNodeDesc>> filters;
+ private List<ExprNodeDesc> residualFilterExprs;
+
// pos of outer join alias=<pos of other alias:num of filters on outer join alias>xn
// for example,
// a left outer join b on a.k=b.k AND a.k>5 full outer join c on a.k=c.k AND a.k>10 AND c.k>20
@@ -193,6 +195,7 @@ public class JoinDesc extends AbstractOperatorDesc {
this.tagOrder = clone.tagOrder;
this.filters = clone.filters;
this.filterMap = clone.filterMap;
+ this.residualFilterExprs = clone.residualFilterExprs;
this.statistics = clone.statistics;
}
@@ -294,6 +297,36 @@ public class JoinDesc extends AbstractOperatorDesc {
this.filters = filters;
}
+ @Explain(displayName = "residual filter predicates")
+ public String getResidualFilterExprsString() {
+ if (getResidualFilterExprs() == null || getResidualFilterExprs().size() == 0) {
+ return null;
+ }
+
+ StringBuilder sb = new StringBuilder();
+ boolean first = true;
+ for (ExprNodeDesc expr : getResidualFilterExprs()) {
+ if (!first) {
+ sb.append(" ");
+ }
+
+ first = false;
+ sb.append("{");
+ sb.append(expr.getExprString());
+ sb.append("}");
+ }
+
+ return sb.toString();
+ }
+
+ public List<ExprNodeDesc> getResidualFilterExprs() {
+ return residualFilterExprs;
+ }
+
+ public void setResidualFilterExprs(List<ExprNodeDesc> residualFilterExprs) {
+ this.residualFilterExprs = residualFilterExprs;
+ }
+
@Explain(displayName = "outputColumnNames")
public List<String> getOutputColumnNames() {
return outputColumnNames;
http://git-wip-us.apache.org/repos/asf/hive/blob/9b2badc3/ql/src/test/queries/clientnegative/join45.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientnegative/join45.q b/ql/src/test/queries/clientnegative/join45.q
deleted file mode 100644
index 4e8db96..0000000
--- a/ql/src/test/queries/clientnegative/join45.q
+++ /dev/null
@@ -1,13 +0,0 @@
-set hive.strict.checks.cartesian.product=false;
-
--- SORT_QUERY_RESULTS
-
-CREATE TABLE mytable(val1 INT, val2 INT, val3 INT);
-
--- Outer join with complex pred: not supported
-EXPLAIN
-SELECT *
-FROM mytable src1 LEFT OUTER JOIN mytable src2
-ON (src1.val1+src2.val1>= 2450816
- AND src1.val1+src2.val1<= 2451500);
-
http://git-wip-us.apache.org/repos/asf/hive/blob/9b2badc3/ql/src/test/queries/clientpositive/join46.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/join46.q b/ql/src/test/queries/clientpositive/join46.q
new file mode 100644
index 0000000..e9e9850
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/join46.q
@@ -0,0 +1,263 @@
+set hive.strict.checks.cartesian.product=false;
+set hive.join.emit.interval=2;
+
+CREATE TABLE test1 (key INT, value INT, col_1 STRING);
+INSERT INTO test1 VALUES (NULL, NULL, 'None'), (98, NULL, 'None'),
+ (99, 0, 'Alice'), (99, 2, 'Mat'), (100, 1, 'Bob'), (101, 2, 'Car');
+
+CREATE TABLE test2 (key INT, value INT, col_2 STRING);
+INSERT INTO test2 VALUES (102, 2, 'Del'), (103, 2, 'Ema'),
+ (104, 3, 'Fli'), (105, NULL, 'None');
+
+
+-- Basic outer join
+EXPLAIN
+SELECT *
+FROM test1 LEFT OUTER JOIN test2
+ON (test1.value=test2.value);
+
+SELECT *
+FROM test1 LEFT OUTER JOIN test2
+ON (test1.value=test2.value);
+
+-- Conjunction with pred on multiple inputs and single inputs (left outer join)
+EXPLAIN
+SELECT *
+FROM test1 LEFT OUTER JOIN test2
+ON (test1.value=test2.value
+ AND test1.key between 100 and 102
+ AND test2.key between 100 and 102);
+
+SELECT *
+FROM test1 LEFT OUTER JOIN test2
+ON (test1.value=test2.value
+ AND test1.key between 100 and 102
+ AND test2.key between 100 and 102);
+
+-- Conjunction with pred on single inputs (left outer join)
+EXPLAIN
+SELECT *
+FROM test1 LEFT OUTER JOIN test2
+ON (test1.key between 100 and 102
+ AND test2.key between 100 and 102);
+
+SELECT *
+FROM test1 LEFT OUTER JOIN test2
+ON (test1.key between 100 and 102
+ AND test2.key between 100 and 102);
+
+-- Conjunction with pred on multiple inputs and none (left outer join)
+EXPLAIN
+SELECT *
+FROM test1 RIGHT OUTER JOIN test2
+ON (test1.value=test2.value AND true);
+
+SELECT *
+FROM test1 RIGHT OUTER JOIN test2
+ON (test1.value=test2.value AND true);
+
+-- Condition on one input (left outer join)
+EXPLAIN
+SELECT *
+FROM test1 LEFT OUTER JOIN test2
+ON (test1.key between 100 and 102);
+
+SELECT *
+FROM test1 LEFT OUTER JOIN test2
+ON (test1.key between 100 and 102);
+
+-- Disjunction with pred on multiple inputs and single inputs (left outer join)
+EXPLAIN
+SELECT *
+FROM test1 LEFT OUTER JOIN test2
+ON (test1.value=test2.value
+ OR test1.key between 100 and 102
+ OR test2.key between 100 and 102);
+
+SELECT *
+FROM test1 LEFT OUTER JOIN test2
+ON (test1.value=test2.value
+ OR test1.key between 100 and 102
+ OR test2.key between 100 and 102);
+
+-- Disjunction with pred on multiple inputs and left input (left outer join)
+EXPLAIN
+SELECT *
+FROM test1 LEFT OUTER JOIN test2
+ON (test1.value=test2.value
+ OR test1.key between 100 and 102);
+
+SELECT *
+FROM test1 LEFT OUTER JOIN test2
+ON (test1.value=test2.value
+ OR test1.key between 100 and 102);
+
+-- Disjunction with pred on multiple inputs and right input (left outer join)
+EXPLAIN
+SELECT *
+FROM test1 LEFT OUTER JOIN test2
+ON (test1.value=test2.value
+ OR test2.key between 100 and 102);
+
+SELECT *
+FROM test1 LEFT OUTER JOIN test2
+ON (test1.value=test2.value
+ OR test2.key between 100 and 102);
+
+-- Keys plus residual (left outer join)
+EXPLAIN
+SELECT *
+FROM test1 LEFT OUTER JOIN test2
+ON (test1.value=test2.value
+ AND (test1.key between 100 and 102
+ OR test2.key between 100 and 102));
+
+SELECT *
+FROM test1 LEFT OUTER JOIN test2
+ON (test1.value=test2.value
+ AND (test1.key between 100 and 102
+ OR test2.key between 100 and 102));
+
+-- Disjunction with pred on multiple inputs and single inputs (right outer join)
+EXPLAIN
+SELECT *
+FROM test1 RIGHT OUTER JOIN test2
+ON (test1.value=test2.value
+ OR test1.key between 100 and 102
+ OR test2.key between 100 and 102);
+
+SELECT *
+FROM test1 RIGHT OUTER JOIN test2
+ON (test1.value=test2.value
+ OR test1.key between 100 and 102
+ OR test2.key between 100 and 102);
+
+-- Disjunction with pred on multiple inputs and left input (right outer join)
+EXPLAIN
+SELECT *
+FROM test1 RIGHT OUTER JOIN test2
+ON (test1.value=test2.value
+ OR test1.key between 100 and 102);
+
+SELECT *
+FROM test1 RIGHT OUTER JOIN test2
+ON (test1.value=test2.value
+ OR test1.key between 100 and 102);
+
+-- Disjunction with pred on multiple inputs and right input (right outer join)
+EXPLAIN
+SELECT *
+FROM test1 RIGHT OUTER JOIN test2
+ON (test1.value=test2.value
+ OR test2.key between 100 and 102);
+
+SELECT *
+FROM test1 RIGHT OUTER JOIN test2
+ON (test1.value=test2.value
+ OR test2.key between 100 and 102);
+
+-- Keys plus residual (right outer join)
+EXPLAIN
+SELECT *
+FROM test1 RIGHT OUTER JOIN test2
+ON (test1.value=test2.value
+ AND (test1.key between 100 and 102
+ OR test2.key between 100 and 102));
+
+SELECT *
+FROM test1 RIGHT OUTER JOIN test2
+ON (test1.value=test2.value
+ AND (test1.key between 100 and 102
+ OR test2.key between 100 and 102));
+
+-- Disjunction with pred on multiple inputs and single inputs (full outer join)
+EXPLAIN
+SELECT *
+FROM test1 FULL OUTER JOIN test2
+ON (test1.value=test2.value
+ OR test1.key between 100 and 102
+ OR test2.key between 100 and 102);
+
+SELECT *
+FROM test1 FULL OUTER JOIN test2
+ON (test1.value=test2.value
+ OR test1.key between 100 and 102
+ OR test2.key between 100 and 102);
+
+-- Disjunction with pred on multiple inputs and left input (full outer join)
+EXPLAIN
+SELECT *
+FROM test1 FULL OUTER JOIN test2
+ON (test1.value=test2.value
+ OR test1.key between 100 and 102);
+
+SELECT *
+FROM test1 FULL OUTER JOIN test2
+ON (test1.value=test2.value
+ OR test1.key between 100 and 102);
+
+-- Disjunction with pred on multiple inputs and right input (full outer join)
+EXPLAIN
+SELECT *
+FROM test1 FULL OUTER JOIN test2
+ON (test1.value=test2.value
+ OR test2.key between 100 and 102);
+
+SELECT *
+FROM test1 FULL OUTER JOIN test2
+ON (test1.value=test2.value
+ OR test2.key between 100 and 102);
+
+-- Keys plus residual (full outer join)
+EXPLAIN
+SELECT *
+FROM test1 FULL OUTER JOIN test2
+ON (test1.value=test2.value
+ AND (test1.key between 100 and 102
+ OR test2.key between 100 and 102));
+
+SELECT *
+FROM test1 FULL OUTER JOIN test2
+ON (test1.value=test2.value
+ AND (test1.key between 100 and 102
+ OR test2.key between 100 and 102));
+
+-- Mixed ( FOJ (ROJ, LOJ) )
+EXPLAIN
+SELECT *
+FROM (
+ SELECT test1.key AS key1, test1.value AS value1, test1.col_1 AS col_1,
+ test2.key AS key2, test2.value AS value2, test2.col_2 AS col_2
+ FROM test1 RIGHT OUTER JOIN test2
+ ON (test1.value=test2.value
+ AND (test1.key between 100 and 102
+ OR test2.key between 100 and 102))
+ ) sq1
+FULL OUTER JOIN (
+ SELECT test1.key AS key3, test1.value AS value3, test1.col_1 AS col_3,
+ test2.key AS key4, test2.value AS value4, test2.col_2 AS col_4
+ FROM test1 LEFT OUTER JOIN test2
+ ON (test1.value=test2.value
+ AND (test1.key between 100 and 102
+ OR test2.key between 100 and 102))
+ ) sq2
+ON (sq1.value1 is null or sq2.value4 is null and sq2.value3 != sq1.value2);
+
+SELECT *
+FROM (
+ SELECT test1.key AS key1, test1.value AS value1, test1.col_1 AS col_1,
+ test2.key AS key2, test2.value AS value2, test2.col_2 AS col_2
+ FROM test1 RIGHT OUTER JOIN test2
+ ON (test1.value=test2.value
+ AND (test1.key between 100 and 102
+ OR test2.key between 100 and 102))
+ ) sq1
+FULL OUTER JOIN (
+ SELECT test1.key AS key3, test1.value AS value3, test1.col_1 AS col_3,
+ test2.key AS key4, test2.value AS value4, test2.col_2 AS col_4
+ FROM test1 LEFT OUTER JOIN test2
+ ON (test1.value=test2.value
+ AND (test1.key between 100 and 102
+ OR test2.key between 100 and 102))
+ ) sq2
+ON (sq1.value1 is null or sq2.value4 is null and sq2.value3 != sq1.value2);
http://git-wip-us.apache.org/repos/asf/hive/blob/9b2badc3/ql/src/test/queries/clientpositive/mapjoin46.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/mapjoin46.q b/ql/src/test/queries/clientpositive/mapjoin46.q
new file mode 100644
index 0000000..348dd67
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/mapjoin46.q
@@ -0,0 +1,264 @@
+set hive.auto.convert.join=true;
+set hive.strict.checks.cartesian.product=false;
+set hive.join.emit.interval=2;
+
+CREATE TABLE test1 (key INT, value INT, col_1 STRING);
+INSERT INTO test1 VALUES (NULL, NULL, 'None'), (98, NULL, 'None'),
+ (99, 0, 'Alice'), (99, 2, 'Mat'), (100, 1, 'Bob'), (101, 2, 'Car');
+
+CREATE TABLE test2 (key INT, value INT, col_2 STRING);
+INSERT INTO test2 VALUES (102, 2, 'Del'), (103, 2, 'Ema'),
+ (104, 3, 'Fli'), (105, NULL, 'None');
+
+
+-- Basic outer join
+EXPLAIN
+SELECT *
+FROM test1 LEFT OUTER JOIN test2
+ON (test1.value=test2.value);
+
+SELECT *
+FROM test1 LEFT OUTER JOIN test2
+ON (test1.value=test2.value);
+
+-- Conjunction with pred on multiple inputs and single inputs (left outer join)
+EXPLAIN
+SELECT *
+FROM test1 LEFT OUTER JOIN test2
+ON (test1.value=test2.value
+ AND test1.key between 100 and 102
+ AND test2.key between 100 and 102);
+
+SELECT *
+FROM test1 LEFT OUTER JOIN test2
+ON (test1.value=test2.value
+ AND test1.key between 100 and 102
+ AND test2.key between 100 and 102);
+
+-- Conjunction with pred on single inputs (left outer join)
+EXPLAIN
+SELECT *
+FROM test1 LEFT OUTER JOIN test2
+ON (test1.key between 100 and 102
+ AND test2.key between 100 and 102);
+
+SELECT *
+FROM test1 LEFT OUTER JOIN test2
+ON (test1.key between 100 and 102
+ AND test2.key between 100 and 102);
+
+-- Conjunction with pred on multiple inputs and none (left outer join)
+EXPLAIN
+SELECT *
+FROM test1 RIGHT OUTER JOIN test2
+ON (test1.value=test2.value AND true);
+
+SELECT *
+FROM test1 RIGHT OUTER JOIN test2
+ON (test1.value=test2.value AND true);
+
+-- Condition on one input (left outer join)
+EXPLAIN
+SELECT *
+FROM test1 LEFT OUTER JOIN test2
+ON (test1.key between 100 and 102);
+
+SELECT *
+FROM test1 LEFT OUTER JOIN test2
+ON (test1.key between 100 and 102);
+
+-- Disjunction with pred on multiple inputs and single inputs (left outer join)
+EXPLAIN
+SELECT *
+FROM test1 LEFT OUTER JOIN test2
+ON (test1.value=test2.value
+ OR test1.key between 100 and 102
+ OR test2.key between 100 and 102);
+
+SELECT *
+FROM test1 LEFT OUTER JOIN test2
+ON (test1.value=test2.value
+ OR test1.key between 100 and 102
+ OR test2.key between 100 and 102);
+
+-- Disjunction with pred on multiple inputs and left input (left outer join)
+EXPLAIN
+SELECT *
+FROM test1 LEFT OUTER JOIN test2
+ON (test1.value=test2.value
+ OR test1.key between 100 and 102);
+
+SELECT *
+FROM test1 LEFT OUTER JOIN test2
+ON (test1.value=test2.value
+ OR test1.key between 100 and 102);
+
+-- Disjunction with pred on multiple inputs and right input (left outer join)
+EXPLAIN
+SELECT *
+FROM test1 LEFT OUTER JOIN test2
+ON (test1.value=test2.value
+ OR test2.key between 100 and 102);
+
+SELECT *
+FROM test1 LEFT OUTER JOIN test2
+ON (test1.value=test2.value
+ OR test2.key between 100 and 102);
+
+-- Keys plus residual (left outer join)
+EXPLAIN
+SELECT *
+FROM test1 LEFT OUTER JOIN test2
+ON (test1.value=test2.value
+ AND (test1.key between 100 and 102
+ OR test2.key between 100 and 102));
+
+SELECT *
+FROM test1 LEFT OUTER JOIN test2
+ON (test1.value=test2.value
+ AND (test1.key between 100 and 102
+ OR test2.key between 100 and 102));
+
+-- Disjunction with pred on multiple inputs and single inputs (right outer join)
+EXPLAIN
+SELECT *
+FROM test1 RIGHT OUTER JOIN test2
+ON (test1.value=test2.value
+ OR test1.key between 100 and 102
+ OR test2.key between 100 and 102);
+
+SELECT *
+FROM test1 RIGHT OUTER JOIN test2
+ON (test1.value=test2.value
+ OR test1.key between 100 and 102
+ OR test2.key between 100 and 102);
+
+-- Disjunction with pred on multiple inputs and left input (right outer join)
+EXPLAIN
+SELECT *
+FROM test1 RIGHT OUTER JOIN test2
+ON (test1.value=test2.value
+ OR test1.key between 100 and 102);
+
+SELECT *
+FROM test1 RIGHT OUTER JOIN test2
+ON (test1.value=test2.value
+ OR test1.key between 100 and 102);
+
+-- Disjunction with pred on multiple inputs and right input (right outer join)
+EXPLAIN
+SELECT *
+FROM test1 RIGHT OUTER JOIN test2
+ON (test1.value=test2.value
+ OR test2.key between 100 and 102);
+
+SELECT *
+FROM test1 RIGHT OUTER JOIN test2
+ON (test1.value=test2.value
+ OR test2.key between 100 and 102);
+
+-- Keys plus residual (right outer join)
+EXPLAIN
+SELECT *
+FROM test1 RIGHT OUTER JOIN test2
+ON (test1.value=test2.value
+ AND (test1.key between 100 and 102
+ OR test2.key between 100 and 102));
+
+SELECT *
+FROM test1 RIGHT OUTER JOIN test2
+ON (test1.value=test2.value
+ AND (test1.key between 100 and 102
+ OR test2.key between 100 and 102));
+
+-- Disjunction with pred on multiple inputs and single inputs (full outer join)
+EXPLAIN
+SELECT *
+FROM test1 FULL OUTER JOIN test2
+ON (test1.value=test2.value
+ OR test1.key between 100 and 102
+ OR test2.key between 100 and 102);
+
+SELECT *
+FROM test1 FULL OUTER JOIN test2
+ON (test1.value=test2.value
+ OR test1.key between 100 and 102
+ OR test2.key between 100 and 102);
+
+-- Disjunction with pred on multiple inputs and left input (full outer join)
+EXPLAIN
+SELECT *
+FROM test1 FULL OUTER JOIN test2
+ON (test1.value=test2.value
+ OR test1.key between 100 and 102);
+
+SELECT *
+FROM test1 FULL OUTER JOIN test2
+ON (test1.value=test2.value
+ OR test1.key between 100 and 102);
+
+-- Disjunction with pred on multiple inputs and right input (full outer join)
+EXPLAIN
+SELECT *
+FROM test1 FULL OUTER JOIN test2
+ON (test1.value=test2.value
+ OR test2.key between 100 and 102);
+
+SELECT *
+FROM test1 FULL OUTER JOIN test2
+ON (test1.value=test2.value
+ OR test2.key between 100 and 102);
+
+-- Keys plus residual (full outer join)
+EXPLAIN
+SELECT *
+FROM test1 FULL OUTER JOIN test2
+ON (test1.value=test2.value
+ AND (test1.key between 100 and 102
+ OR test2.key between 100 and 102));
+
+SELECT *
+FROM test1 FULL OUTER JOIN test2
+ON (test1.value=test2.value
+ AND (test1.key between 100 and 102
+ OR test2.key between 100 and 102));
+
+-- Mixed ( FOJ (ROJ, LOJ) )
+EXPLAIN
+SELECT *
+FROM (
+ SELECT test1.key AS key1, test1.value AS value1, test1.col_1 AS col_1,
+ test2.key AS key2, test2.value AS value2, test2.col_2 AS col_2
+ FROM test1 RIGHT OUTER JOIN test2
+ ON (test1.value=test2.value
+ AND (test1.key between 100 and 102
+ OR test2.key between 100 and 102))
+ ) sq1
+FULL OUTER JOIN (
+ SELECT test1.key AS key3, test1.value AS value3, test1.col_1 AS col_3,
+ test2.key AS key4, test2.value AS value4, test2.col_2 AS col_4
+ FROM test1 LEFT OUTER JOIN test2
+ ON (test1.value=test2.value
+ AND (test1.key between 100 and 102
+ OR test2.key between 100 and 102))
+ ) sq2
+ON (sq1.value1 is null or sq2.value4 is null and sq2.value3 != sq1.value2);
+
+SELECT *
+FROM (
+ SELECT test1.key AS key1, test1.value AS value1, test1.col_1 AS col_1,
+ test2.key AS key2, test2.value AS value2, test2.col_2 AS col_2
+ FROM test1 RIGHT OUTER JOIN test2
+ ON (test1.value=test2.value
+ AND (test1.key between 100 and 102
+ OR test2.key between 100 and 102))
+ ) sq1
+FULL OUTER JOIN (
+ SELECT test1.key AS key3, test1.value AS value3, test1.col_1 AS col_3,
+ test2.key AS key4, test2.value AS value4, test2.col_2 AS col_4
+ FROM test1 LEFT OUTER JOIN test2
+ ON (test1.value=test2.value
+ AND (test1.key between 100 and 102
+ OR test2.key between 100 and 102))
+ ) sq2
+ON (sq1.value1 is null or sq2.value4 is null and sq2.value3 != sq1.value2);
http://git-wip-us.apache.org/repos/asf/hive/blob/9b2badc3/ql/src/test/queries/clientpositive/smb_mapjoin_46.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/smb_mapjoin_46.q b/ql/src/test/queries/clientpositive/smb_mapjoin_46.q
new file mode 100644
index 0000000..ad29414
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/smb_mapjoin_46.q
@@ -0,0 +1,240 @@
+set hive.strict.checks.cartesian.product=false;
+set hive.auto.convert.join=true;
+set hive.auto.convert.sortmerge.join=true;
+set hive.optimize.bucketmapjoin = true;
+set hive.optimize.bucketmapjoin.sortedmerge = true;
+set hive.input.format = org.apache.hadoop.hive.ql.io.BucketizedHiveInputFormat;
+set hive.join.emit.interval=2;
+set hive.exec.reducers.max = 1;
+set hive.merge.mapfiles=false;
+set hive.merge.mapredfiles=false;
+
+CREATE TABLE aux1 (key INT, value INT, col_1 STRING);
+INSERT INTO aux1 VALUES (NULL, NULL, 'None'), (98, NULL, 'None'),
+ (99, 0, 'Alice'), (99, 2, 'Mat'), (100, 1, 'Bob'), (101, 2, 'Car');
+
+CREATE TABLE test1 (key INT, value INT, col_1 STRING) CLUSTERED BY (value) SORTED BY (value) INTO 2 BUCKETS;
+INSERT OVERWRITE TABLE test1
+SELECT * FROM aux1;
+
+CREATE TABLE aux2 (key INT, value INT, col_2 STRING);
+INSERT INTO aux2 VALUES (102, 2, 'Del'), (103, 2, 'Ema'),
+ (104, 3, 'Fli'), (105, NULL, 'None');
+
+CREATE TABLE test2 (key INT, value INT, col_2 STRING) CLUSTERED BY (value) SORTED BY (value) INTO 2 BUCKETS;
+INSERT OVERWRITE TABLE test2
+SELECT * FROM aux2;
+
+
+-- Basic outer join
+EXPLAIN
+SELECT *
+FROM test1 LEFT OUTER JOIN test2
+ON (test1.value=test2.value);
+
+SELECT *
+FROM test1 LEFT OUTER JOIN test2
+ON (test1.value=test2.value);
+
+-- Conjunction with pred on multiple inputs and single inputs (left outer join)
+EXPLAIN
+SELECT *
+FROM test1 LEFT OUTER JOIN test2
+ON (test1.value=test2.value
+ AND test1.key between 100 and 102
+ AND test2.key between 100 and 102);
+
+SELECT *
+FROM test1 LEFT OUTER JOIN test2
+ON (test1.value=test2.value
+ AND test1.key between 100 and 102
+ AND test2.key between 100 and 102);
+
+-- Conjunction with pred on single inputs (left outer join)
+EXPLAIN
+SELECT *
+FROM test1 LEFT OUTER JOIN test2
+ON (test1.key between 100 and 102
+ AND test2.key between 100 and 102);
+
+SELECT *
+FROM test1 LEFT OUTER JOIN test2
+ON (test1.key between 100 and 102
+ AND test2.key between 100 and 102);
+
+-- Conjunction with pred on multiple inputs and none (left outer join)
+EXPLAIN
+SELECT *
+FROM test1 RIGHT OUTER JOIN test2
+ON (test1.value=test2.value AND true);
+
+SELECT *
+FROM test1 RIGHT OUTER JOIN test2
+ON (test1.value=test2.value AND true);
+
+-- Condition on one input (left outer join)
+EXPLAIN
+SELECT *
+FROM test1 LEFT OUTER JOIN test2
+ON (test1.key between 100 and 102);
+
+SELECT *
+FROM test1 LEFT OUTER JOIN test2
+ON (test1.key between 100 and 102);
+
+-- Disjunction with pred on multiple inputs and single inputs (left outer join)
+EXPLAIN
+SELECT *
+FROM test1 LEFT OUTER JOIN test2
+ON (test1.value=test2.value
+ OR test1.key between 100 and 102
+ OR test2.key between 100 and 102);
+
+SELECT *
+FROM test1 LEFT OUTER JOIN test2
+ON (test1.value=test2.value
+ OR test1.key between 100 and 102
+ OR test2.key between 100 and 102);
+
+-- Disjunction with pred on multiple inputs and left input (left outer join)
+EXPLAIN
+SELECT *
+FROM test1 LEFT OUTER JOIN test2
+ON (test1.value=test2.value
+ OR test1.key between 100 and 102);
+
+SELECT *
+FROM test1 LEFT OUTER JOIN test2
+ON (test1.value=test2.value
+ OR test1.key between 100 and 102);
+
+-- Disjunction with pred on multiple inputs and right input (left outer join)
+EXPLAIN
+SELECT *
+FROM test1 LEFT OUTER JOIN test2
+ON (test1.value=test2.value
+ OR test2.key between 100 and 102);
+
+SELECT *
+FROM test1 LEFT OUTER JOIN test2
+ON (test1.value=test2.value
+ OR test2.key between 100 and 102);
+
+-- Keys plus residual (left outer join)
+EXPLAIN
+SELECT *
+FROM test1 LEFT OUTER JOIN test2
+ON (test1.value=test2.value
+ AND (test1.key between 100 and 102
+ OR test2.key between 100 and 102));
+
+SELECT *
+FROM test1 LEFT OUTER JOIN test2
+ON (test1.value=test2.value
+ AND (test1.key between 100 and 102
+ OR test2.key between 100 and 102));
+
+-- Disjunction with pred on multiple inputs and single inputs (right outer join)
+EXPLAIN
+SELECT *
+FROM test1 RIGHT OUTER JOIN test2
+ON (test1.value=test2.value
+ OR test1.key between 100 and 102
+ OR test2.key between 100 and 102);
+
+SELECT *
+FROM test1 RIGHT OUTER JOIN test2
+ON (test1.value=test2.value
+ OR test1.key between 100 and 102
+ OR test2.key between 100 and 102);
+
+-- Disjunction with pred on multiple inputs and left input (right outer join)
+EXPLAIN
+SELECT *
+FROM test1 RIGHT OUTER JOIN test2
+ON (test1.value=test2.value
+ OR test1.key between 100 and 102);
+
+SELECT *
+FROM test1 RIGHT OUTER JOIN test2
+ON (test1.value=test2.value
+ OR test1.key between 100 and 102);
+
+-- Disjunction with pred on multiple inputs and right input (right outer join)
+EXPLAIN
+SELECT *
+FROM test1 RIGHT OUTER JOIN test2
+ON (test1.value=test2.value
+ OR test2.key between 100 and 102);
+
+SELECT *
+FROM test1 RIGHT OUTER JOIN test2
+ON (test1.value=test2.value
+ OR test2.key between 100 and 102);
+
+-- Keys plus residual (right outer join)
+EXPLAIN
+SELECT *
+FROM test1 RIGHT OUTER JOIN test2
+ON (test1.value=test2.value
+ AND (test1.key between 100 and 102
+ OR test2.key between 100 and 102));
+
+SELECT *
+FROM test1 RIGHT OUTER JOIN test2
+ON (test1.value=test2.value
+ AND (test1.key between 100 and 102
+ OR test2.key between 100 and 102));
+
+-- Disjunction with pred on multiple inputs and single inputs (full outer join)
+EXPLAIN
+SELECT *
+FROM test1 FULL OUTER JOIN test2
+ON (test1.value=test2.value
+ OR test1.key between 100 and 102
+ OR test2.key between 100 and 102);
+
+SELECT *
+FROM test1 FULL OUTER JOIN test2
+ON (test1.value=test2.value
+ OR test1.key between 100 and 102
+ OR test2.key between 100 and 102);
+
+-- Disjunction with pred on multiple inputs and left input (full outer join)
+EXPLAIN
+SELECT *
+FROM test1 FULL OUTER JOIN test2
+ON (test1.value=test2.value
+ OR test1.key between 100 and 102);
+
+SELECT *
+FROM test1 FULL OUTER JOIN test2
+ON (test1.value=test2.value
+ OR test1.key between 100 and 102);
+
+-- Disjunction with pred on multiple inputs and right input (full outer join)
+EXPLAIN
+SELECT *
+FROM test1 FULL OUTER JOIN test2
+ON (test1.value=test2.value
+ OR test2.key between 100 and 102);
+
+SELECT *
+FROM test1 FULL OUTER JOIN test2
+ON (test1.value=test2.value
+ OR test2.key between 100 and 102);
+
+-- Keys plus residual (full outer join)
+EXPLAIN
+SELECT *
+FROM test1 FULL OUTER JOIN test2
+ON (test1.value=test2.value
+ AND (test1.key between 100 and 102
+ OR test2.key between 100 and 102));
+
+SELECT *
+FROM test1 FULL OUTER JOIN test2
+ON (test1.value=test2.value
+ AND (test1.key between 100 and 102
+ OR test2.key between 100 and 102));
+
http://git-wip-us.apache.org/repos/asf/hive/blob/9b2badc3/ql/src/test/queries/clientpositive/vectorized_join46.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/vectorized_join46.q b/ql/src/test/queries/clientpositive/vectorized_join46.q
new file mode 100644
index 0000000..a0eb1b2
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/vectorized_join46.q
@@ -0,0 +1,225 @@
+set hive.vectorized.execution.enabled=true;
+set hive.auto.convert.join=true;
+set hive.strict.checks.cartesian.product=false;
+set hive.join.emit.interval=2;
+
+CREATE TABLE test1 (key INT, value INT, col_1 STRING);
+INSERT INTO test1 VALUES (NULL, NULL, 'None'), (98, NULL, 'None'),
+ (99, 0, 'Alice'), (99, 2, 'Mat'), (100, 1, 'Bob'), (101, 2, 'Car');
+
+CREATE TABLE test2 (key INT, value INT, col_2 STRING);
+INSERT INTO test2 VALUES (102, 2, 'Del'), (103, 2, 'Ema'),
+ (104, 3, 'Fli'), (105, NULL, 'None');
+
+
+-- Basic outer join
+EXPLAIN
+SELECT *
+FROM test1 LEFT OUTER JOIN test2
+ON (test1.value=test2.value);
+
+SELECT *
+FROM test1 LEFT OUTER JOIN test2
+ON (test1.value=test2.value);
+
+-- Conjunction with pred on multiple inputs and single inputs (left outer join)
+EXPLAIN
+SELECT *
+FROM test1 LEFT OUTER JOIN test2
+ON (test1.value=test2.value
+ AND test1.key between 100 and 102
+ AND test2.key between 100 and 102);
+
+SELECT *
+FROM test1 LEFT OUTER JOIN test2
+ON (test1.value=test2.value
+ AND test1.key between 100 and 102
+ AND test2.key between 100 and 102);
+
+-- Conjunction with pred on single inputs (left outer join)
+EXPLAIN
+SELECT *
+FROM test1 LEFT OUTER JOIN test2
+ON (test1.key between 100 and 102
+ AND test2.key between 100 and 102);
+
+SELECT *
+FROM test1 LEFT OUTER JOIN test2
+ON (test1.key between 100 and 102
+ AND test2.key between 100 and 102);
+
+-- Conjunction with pred on multiple inputs and none (left outer join)
+EXPLAIN
+SELECT *
+FROM test1 RIGHT OUTER JOIN test2
+ON (test1.value=test2.value AND true);
+
+SELECT *
+FROM test1 RIGHT OUTER JOIN test2
+ON (test1.value=test2.value AND true);
+
+-- Condition on one input (left outer join)
+EXPLAIN
+SELECT *
+FROM test1 LEFT OUTER JOIN test2
+ON (test1.key between 100 and 102);
+
+SELECT *
+FROM test1 LEFT OUTER JOIN test2
+ON (test1.key between 100 and 102);
+
+-- Disjunction with pred on multiple inputs and single inputs (left outer join)
+EXPLAIN
+SELECT *
+FROM test1 LEFT OUTER JOIN test2
+ON (test1.value=test2.value
+ OR test1.key between 100 and 102
+ OR test2.key between 100 and 102);
+
+SELECT *
+FROM test1 LEFT OUTER JOIN test2
+ON (test1.value=test2.value
+ OR test1.key between 100 and 102
+ OR test2.key between 100 and 102);
+
+-- Disjunction with pred on multiple inputs and left input (left outer join)
+EXPLAIN
+SELECT *
+FROM test1 LEFT OUTER JOIN test2
+ON (test1.value=test2.value
+ OR test1.key between 100 and 102);
+
+SELECT *
+FROM test1 LEFT OUTER JOIN test2
+ON (test1.value=test2.value
+ OR test1.key between 100 and 102);
+
+-- Disjunction with pred on multiple inputs and right input (left outer join)
+EXPLAIN
+SELECT *
+FROM test1 LEFT OUTER JOIN test2
+ON (test1.value=test2.value
+ OR test2.key between 100 and 102);
+
+SELECT *
+FROM test1 LEFT OUTER JOIN test2
+ON (test1.value=test2.value
+ OR test2.key between 100 and 102);
+
+-- Keys plus residual (left outer join)
+EXPLAIN
+SELECT *
+FROM test1 LEFT OUTER JOIN test2
+ON (test1.value=test2.value
+ AND (test1.key between 100 and 102
+ OR test2.key between 100 and 102));
+
+SELECT *
+FROM test1 LEFT OUTER JOIN test2
+ON (test1.value=test2.value
+ AND (test1.key between 100 and 102
+ OR test2.key between 100 and 102));
+
+-- Disjunction with pred on multiple inputs and single inputs (right outer join)
+EXPLAIN
+SELECT *
+FROM test1 RIGHT OUTER JOIN test2
+ON (test1.value=test2.value
+ OR test1.key between 100 and 102
+ OR test2.key between 100 and 102);
+
+SELECT *
+FROM test1 RIGHT OUTER JOIN test2
+ON (test1.value=test2.value
+ OR test1.key between 100 and 102
+ OR test2.key between 100 and 102);
+
+-- Disjunction with pred on multiple inputs and left input (right outer join)
+EXPLAIN
+SELECT *
+FROM test1 RIGHT OUTER JOIN test2
+ON (test1.value=test2.value
+ OR test1.key between 100 and 102);
+
+SELECT *
+FROM test1 RIGHT OUTER JOIN test2
+ON (test1.value=test2.value
+ OR test1.key between 100 and 102);
+
+-- Disjunction with pred on multiple inputs and right input (right outer join)
+EXPLAIN
+SELECT *
+FROM test1 RIGHT OUTER JOIN test2
+ON (test1.value=test2.value
+ OR test2.key between 100 and 102);
+
+SELECT *
+FROM test1 RIGHT OUTER JOIN test2
+ON (test1.value=test2.value
+ OR test2.key between 100 and 102);
+
+-- Keys plus residual (right outer join)
+EXPLAIN
+SELECT *
+FROM test1 RIGHT OUTER JOIN test2
+ON (test1.value=test2.value
+ AND (test1.key between 100 and 102
+ OR test2.key between 100 and 102));
+
+SELECT *
+FROM test1 RIGHT OUTER JOIN test2
+ON (test1.value=test2.value
+ AND (test1.key between 100 and 102
+ OR test2.key between 100 and 102));
+
+-- Disjunction with pred on multiple inputs and single inputs (full outer join)
+EXPLAIN
+SELECT *
+FROM test1 FULL OUTER JOIN test2
+ON (test1.value=test2.value
+ OR test1.key between 100 and 102
+ OR test2.key between 100 and 102);
+
+SELECT *
+FROM test1 FULL OUTER JOIN test2
+ON (test1.value=test2.value
+ OR test1.key between 100 and 102
+ OR test2.key between 100 and 102);
+
+-- Disjunction with pred on multiple inputs and left input (full outer join)
+EXPLAIN
+SELECT *
+FROM test1 FULL OUTER JOIN test2
+ON (test1.value=test2.value
+ OR test1.key between 100 and 102);
+
+SELECT *
+FROM test1 FULL OUTER JOIN test2
+ON (test1.value=test2.value
+ OR test1.key between 100 and 102);
+
+-- Disjunction with pred on multiple inputs and right input (full outer join)
+EXPLAIN
+SELECT *
+FROM test1 FULL OUTER JOIN test2
+ON (test1.value=test2.value
+ OR test2.key between 100 and 102);
+
+SELECT *
+FROM test1 FULL OUTER JOIN test2
+ON (test1.value=test2.value
+ OR test2.key between 100 and 102);
+
+-- Keys plus residual (full outer join)
+EXPLAIN
+SELECT *
+FROM test1 FULL OUTER JOIN test2
+ON (test1.value=test2.value
+ AND (test1.key between 100 and 102
+ OR test2.key between 100 and 102));
+
+SELECT *
+FROM test1 FULL OUTER JOIN test2
+ON (test1.value=test2.value
+ AND (test1.key between 100 and 102
+ OR test2.key between 100 and 102));
http://git-wip-us.apache.org/repos/asf/hive/blob/9b2badc3/ql/src/test/results/clientnegative/join45.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientnegative/join45.q.out b/ql/src/test/results/clientnegative/join45.q.out
deleted file mode 100644
index 87ef769..0000000
--- a/ql/src/test/results/clientnegative/join45.q.out
+++ /dev/null
@@ -1,13 +0,0 @@
-PREHOOK: query: -- SORT_QUERY_RESULTS
-
-CREATE TABLE mytable(val1 INT, val2 INT, val3 INT)
-PREHOOK: type: CREATETABLE
-PREHOOK: Output: database:default
-PREHOOK: Output: default@mytable
-POSTHOOK: query: -- SORT_QUERY_RESULTS
-
-CREATE TABLE mytable(val1 INT, val2 INT, val3 INT)
-POSTHOOK: type: CREATETABLE
-POSTHOOK: Output: database:default
-POSTHOOK: Output: default@mytable
-FAILED: SemanticException [Error 10407]: Complex condition not supported for (LEFT|RIGHT|FULL) OUTER JOIN