You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by na...@apache.org on 2010/09/21 23:00:22 UTC
svn commit: r999637 [1/6] - in /hadoop/hive/trunk: ./
common/src/java/org/apache/hadoop/hive/conf/ data/files/
ql/src/java/org/apache/hadoop/hive/ql/exec/
ql/src/java/org/apache/hadoop/hive/ql/optimizer/
ql/src/java/org/apache/hadoop/hive/ql/optimizer/...
Author: namit
Date: Tue Sep 21 21:00:21 2010
New Revision: 999637
URL: http://svn.apache.org/viewvc?rev=999637&view=rev
Log:
HIVE-1534. Join filters do not work correctly with outer joins
(Amareshwari Sriramadasu via namit)
Added:
hadoop/hive/trunk/data/files/in3.txt
hadoop/hive/trunk/ql/src/test/queries/clientpositive/join_filters.q
hadoop/hive/trunk/ql/src/test/results/clientpositive/join_filters.q.out
Modified:
hadoop/hive/trunk/CHANGES.txt
hadoop/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinOperator.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/SMBMapJoinOperator.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/GenMRSkewJoinProcessor.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ErrorMsg.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/QBJoinTree.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/JoinDesc.java
hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/MapJoinDesc.java
hadoop/hive/trunk/ql/src/test/results/clientpositive/join20.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/join21.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/join40.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/louter_join_ppr.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/outer_join_ppr.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/router_join_ppr.q.out
hadoop/hive/trunk/ql/src/test/results/clientpositive/union22.q.out
hadoop/hive/trunk/ql/src/test/results/compiler/plan/join1.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/join2.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/join3.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/join4.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/join5.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/join6.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/join7.q.xml
hadoop/hive/trunk/ql/src/test/results/compiler/plan/join8.q.xml
Modified: hadoop/hive/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/CHANGES.txt?rev=999637&r1=999636&r2=999637&view=diff
==============================================================================
--- hadoop/hive/trunk/CHANGES.txt (original)
+++ hadoop/hive/trunk/CHANGES.txt Tue Sep 21 21:00:21 2010
@@ -282,6 +282,9 @@ Trunk - Unreleased
property is added in run config
(Steven Wong via jvs)
+ HIVE-1534. Join filters do not work correctly with outer joins
+ (Amareshwari Sriramadasu via namit)
+
TESTS
HIVE-1464. improve test query performance
Modified: hadoop/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java?rev=999637&r1=999636&r2=999637&view=diff
==============================================================================
--- hadoop/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (original)
+++ hadoop/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java Tue Sep 21 21:00:21 2010
@@ -286,6 +286,7 @@ public class HiveConf extends Configurat
// For har files
HIVEARCHIVEENABLED("hive.archive.enabled", false),
HIVEHARPARENTDIRSETTABLE("hive.archive.har.parentdir.settable", false),
+ HIVEOUTERJOINSUPPORTSFILTERS("hive.outerjoin.supports.filters", true),
;
Added: hadoop/hive/trunk/data/files/in3.txt
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/data/files/in3.txt?rev=999637&view=auto
==============================================================================
--- hadoop/hive/trunk/data/files/in3.txt (added)
+++ hadoop/hive/trunk/data/files/in3.txt Tue Sep 21 21:00:21 2010
@@ -0,0 +1,4 @@
+1235
+40
+48
+100100
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java?rev=999637&r1=999636&r2=999637&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java Tue Sep 21 21:00:21 2010
@@ -44,8 +44,12 @@ import org.apache.hadoop.hive.serde2.laz
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.ObjectInspectorCopyOption;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
+import org.apache.hadoop.io.BooleanWritable;
import org.apache.hadoop.mapred.SequenceFileInputFormat;
import org.apache.hadoop.util.ReflectionUtils;
@@ -94,17 +98,34 @@ public abstract class CommonJoinOperator
protected transient int numAliases; // number of aliases
/**
- * The expressions for join outputs.
+ * The expressions for join inputs.
*/
protected transient Map<Byte, List<ExprNodeEvaluator>> joinValues;
+
+ /**
+ * The filters for join
+ */
+ protected transient Map<Byte, List<ExprNodeEvaluator>> joinFilters;
+
/**
* The ObjectInspectors for the join inputs.
*/
protected transient Map<Byte, List<ObjectInspector>> joinValuesObjectInspectors;
+
+ /**
+ * The ObjectInspectors for join filters.
+ */
+ protected transient
+ Map<Byte, List<ObjectInspector>> joinFilterObjectInspectors;
/**
* The standard ObjectInspectors for the join inputs.
*/
protected transient Map<Byte, List<ObjectInspector>> joinValuesStandardObjectInspectors;
+ /**
+ * The standard ObjectInspectors for the row container.
+ */
+ protected transient
+ Map<Byte, List<ObjectInspector>> rowContainerStandardObjectInspectors;
protected static transient Byte[] order; // order in which the results should
// be output
@@ -183,6 +204,8 @@ public abstract class CommonJoinOperator
this.posToAliasMap = clone.posToAliasMap;
this.spillTableDesc = clone.spillTableDesc;
this.statsMap = clone.statsMap;
+ this.joinFilters = clone.joinFilters;
+ this.joinFilterObjectInspectors = clone.joinFilterObjectInspectors;
}
protected int populateJoinKeyValue(Map<Byte, List<ExprNodeEvaluator>> outMap,
@@ -282,6 +305,8 @@ public abstract class CommonJoinOperator
joinValues = new HashMap<Byte, List<ExprNodeEvaluator>>();
+ joinFilters = new HashMap<Byte, List<ExprNodeEvaluator>>();
+
if (order == null) {
order = conf.getTagOrder();
}
@@ -289,11 +314,31 @@ public abstract class CommonJoinOperator
noOuterJoin = conf.isNoOuterJoin();
totalSz = populateJoinKeyValue(joinValues, conf.getExprs());
+ populateJoinKeyValue(joinFilters, conf.getFilters());
joinValuesObjectInspectors = getObjectInspectorsFromEvaluators(joinValues,
inputObjInspectors);
+ joinFilterObjectInspectors = getObjectInspectorsFromEvaluators(joinFilters,
+ inputObjInspectors);
joinValuesStandardObjectInspectors = getStandardObjectInspectors(joinValuesObjectInspectors);
+ if (noOuterJoin) {
+ rowContainerStandardObjectInspectors = joinValuesStandardObjectInspectors;
+ } else {
+ Map<Byte, List<ObjectInspector>> rowContainerObjectInspectors =
+ new HashMap<Byte, List<ObjectInspector>>();
+ for (Byte alias : order) {
+ ArrayList<ObjectInspector> rcOIs = new ArrayList<ObjectInspector>();
+ rcOIs.addAll(joinValuesObjectInspectors.get(alias));
+ // for each alias, add object inspector for boolean as the last element
+ rcOIs.add(
+ PrimitiveObjectInspectorFactory.writableBooleanObjectInspector);
+ rowContainerObjectInspectors.put(alias, rcOIs);
+ }
+ rowContainerStandardObjectInspectors =
+ getStandardObjectInspectors(rowContainerObjectInspectors);
+ }
+
dummyObj = new Object[numAliases];
dummyObjVectors = new RowContainer[numAliases];
@@ -313,6 +358,13 @@ public abstract class CommonJoinOperator
for (int j = 0; j < sz; j++) {
nr.add(null);
}
+
+ if (!noOuterJoin) {
+ // add whether the row is filtered or not
+ // this value does not matter for the dummyObj
+ // because the join values are already null
+ nr.add(new BooleanWritable(false));
+ }
dummyObj[pos] = nr;
// there should be only 1 dummy object in the RowContainer
RowContainer<ArrayList<Object>> values = getRowContainer(hconf, pos,
@@ -354,7 +406,7 @@ public abstract class CommonJoinOperator
List<String> colNames = Utilities.getColumnNames(tblDesc.getProperties());
// object inspector for serializing input tuples
rcOI = ObjectInspectorFactory.getStandardStructObjectInspector(colNames,
- joinValuesStandardObjectInspectors.get(pos));
+ rowContainerStandardObjectInspectors.get(pos));
}
rc.setSerDe(serde, rcOI);
@@ -413,6 +465,12 @@ public abstract class CommonJoinOperator
colTypes.append(valueCols.get(k).getTypeString());
colTypes.append(',');
}
+ if (!noOuterJoin) {
+ colNames.append("filtered");
+ colNames.append(',');
+ colTypes.append(TypeInfoFactory.booleanTypeInfo.getTypeName());
+ colTypes.append(',');
+ }
// remove the last ','
colNames.setLength(colNames.length() - 1);
colTypes.setLength(colTypes.length() - 1);
@@ -450,21 +508,45 @@ public abstract class CommonJoinOperator
protected transient Byte alias;
/**
+ * Return the key as a standard object. StandardObject can be inspected by a
+ * standard ObjectInspector.
+ */
+ protected static ArrayList<Object> computeKeys(Object row,
+ List<ExprNodeEvaluator> keyFields, List<ObjectInspector> keyFieldsOI)
+ throws HiveException {
+
+ // Compute the keys
+ ArrayList<Object> nr = new ArrayList<Object>(keyFields.size());
+ for (int i = 0; i < keyFields.size(); i++) {
+
+ nr.add(ObjectInspectorUtils.copyToStandardObject(keyFields.get(i)
+ .evaluate(row), keyFieldsOI.get(i),
+ ObjectInspectorCopyOption.WRITABLE));
+ }
+
+ return nr;
+ }
+
+ /**
* Return the value as a standard object. StandardObject can be inspected by a
* standard ObjectInspector.
*/
protected static ArrayList<Object> computeValues(Object row,
- List<ExprNodeEvaluator> valueFields, List<ObjectInspector> valueFieldsOI)
- throws HiveException {
+ List<ExprNodeEvaluator> valueFields, List<ObjectInspector> valueFieldsOI,
+ List<ExprNodeEvaluator> filters, List<ObjectInspector> filtersOI,
+ boolean noOuterJoin) throws HiveException {
// Compute the values
ArrayList<Object> nr = new ArrayList<Object>(valueFields.size());
for (int i = 0; i < valueFields.size(); i++) {
-
nr.add(ObjectInspectorUtils.copyToStandardObject(valueFields.get(i)
.evaluate(row), valueFieldsOI.get(i),
ObjectInspectorCopyOption.WRITABLE));
}
+ if (!noOuterJoin) {
+ // add whether the row is filtered or not.
+ nr.add(new BooleanWritable(isFiltered(row, filters, filtersOI)));
+ }
return nr;
}
@@ -548,6 +630,12 @@ public abstract class CommonJoinOperator
ArrayList<boolean[]> resNulls, ArrayList<boolean[]> inputNulls,
ArrayList<Object> newObj, IntermediateObject intObj, int left,
boolean newObjNull) {
+ // newObj is null if is already null or
+ // if the row corresponding to the left alias does not pass through filter
+ newObjNull = newObjNull ||
+ ((BooleanWritable) (intObj.getObjs()[left].get(
+ joinValues.get(order[left]).size()))).get();
+
Iterator<boolean[]> nullsIter = inputNulls.iterator();
while (nullsIter.hasNext()) {
boolean[] oldNulls = nullsIter.next();
@@ -593,10 +681,14 @@ public abstract class CommonJoinOperator
}
}
+ // if the row does not pass through filter, all old Objects are null
+ if (((BooleanWritable)newObj.get(newObj.size()-1)).get()) {
+ allOldObjsNull = true;
+ }
nullsIter = inputNulls.iterator();
while (nullsIter.hasNext()) {
boolean[] oldNulls = nullsIter.next();
- boolean oldObjNull = oldNulls[left];
+ boolean oldObjNull = oldNulls[left] || allOldObjsNull;
if (!oldObjNull) {
boolean[] newNulls = new boolean[intObj.getCurSize()];
@@ -652,13 +744,21 @@ public abstract class CommonJoinOperator
break;
}
}
+
+ // if the row does not pass through filter, all old Objects are null
+ if (((BooleanWritable)newObj.get(newObj.size()-1)).get()) {
+ allOldObjsNull = true;
+ }
boolean rhsPreserved = false;
nullsIter = inputNulls.iterator();
while (nullsIter.hasNext()) {
boolean[] oldNulls = nullsIter.next();
- boolean oldObjNull = oldNulls[left];
-
+ // old obj is null even if the row corresponding to the left alias
+ // does not pass through filter
+ boolean oldObjNull = oldNulls[left] || ((BooleanWritable)
+ (intObj.getObjs()[left].get(joinValues.get(order[left]).size()))).get()
+ || allOldObjsNull;
if (!oldObjNull) {
boolean[] newNulls = new boolean[intObj.getCurSize()];
copyOldArray(oldNulls, newNulls);
@@ -901,6 +1001,25 @@ public abstract class CommonJoinOperator
}
/**
+ * Returns true if the row does not pass through filters.
+ */
+ protected static Boolean isFiltered(Object row,
+ List<ExprNodeEvaluator> filters, List<ObjectInspector> ois)
+ throws HiveException {
+ // apply join filters on the row.
+ Boolean ret = false;
+ for (int j = 0; j < filters.size(); j++) {
+ Object condition = filters.get(j).evaluate(row);
+ ret = (Boolean) ((PrimitiveObjectInspector)
+ ois.get(j)).getPrimitiveJavaObject(condition);
+ if (ret == null || !ret) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /**
* All done.
*
*/
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinOperator.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinOperator.java?rev=999637&r1=999636&r2=999637&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinOperator.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinOperator.java Tue Sep 21 21:00:21 2010
@@ -79,7 +79,8 @@ public class JoinOperator extends Common
}
ArrayList<Object> nr = computeValues(row, joinValues.get(alias),
- joinValuesObjectInspectors.get(alias));
+ joinValuesObjectInspectors.get(alias), joinFilters.get(alias),
+ joinFilterObjectInspectors.get(alias), noOuterJoin);
if (handleSkewJoin) {
skewJoinKeyContext.handleSkew(tag);
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java?rev=999637&r1=999636&r2=999637&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java Tue Sep 21 21:00:21 2010
@@ -174,10 +174,11 @@ public class MapJoinOperator extends Abs
}
// compute keys and values as StandardObjects
- ArrayList<Object> key = computeValues(row, joinKeys.get(alias),
+ ArrayList<Object> key = computeKeys(row, joinKeys.get(alias),
joinKeysObjectInspectors.get(alias));
ArrayList<Object> value = computeValues(row, joinValues.get(alias),
- joinValuesObjectInspectors.get(alias));
+ joinValuesObjectInspectors.get(alias), joinFilters.get(alias),
+ joinFilterObjectInspectors.get(alias), noOuterJoin);
// does this source need to be stored in the hash map
if (tag != posBigTable) {
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/SMBMapJoinOperator.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/SMBMapJoinOperator.java?rev=999637&r1=999636&r2=999637&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/SMBMapJoinOperator.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/SMBMapJoinOperator.java Tue Sep 21 21:00:21 2010
@@ -208,10 +208,11 @@ public class SMBMapJoinOperator extends
byte alias = (byte) tag;
// compute keys and values as StandardObjects
- ArrayList<Object> key = computeValues(row, joinKeys.get(alias),
+ ArrayList<Object> key = computeKeys(row, joinKeys.get(alias),
joinKeysObjectInspectors.get(alias));
ArrayList<Object> value = computeValues(row, joinValues.get(alias),
- joinValuesObjectInspectors.get(alias));
+ joinValuesObjectInspectors.get(alias), joinFilters.get(alias),
+ joinFilterObjectInspectors.get(alias), noOuterJoin);
//have we reached a new key group?
boolean nextKeyGroup = processKey(alias, key);
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java?rev=999637&r1=999636&r2=999637&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ColumnPrunerProcFactory.java Tue Sep 21 21:00:21 2010
@@ -206,7 +206,7 @@ public final class ColumnPrunerProcFacto
needed_columns.add(position);
}
}
-
+
desc.setVirtualCols(newVirtualCols);
scanOp.setNeededColumnIDs(needed_columns);
return null;
@@ -600,6 +600,20 @@ public final class ColumnPrunerProcFacto
List<String> childColLists = cppCtx.genColLists(op);
+ //add the columns in join filters
+ Set<Map.Entry<Byte, List<ExprNodeDesc>>> filters =
+ conf.getFilters().entrySet();
+ Iterator<Map.Entry<Byte, List<ExprNodeDesc>>> iter = filters.iterator();
+ while (iter.hasNext()) {
+ Map.Entry<Byte, List<ExprNodeDesc>> entry = iter.next();
+ Byte tag = entry.getKey();
+ for (ExprNodeDesc desc : entry.getValue()) {
+ List<String> cols = prunedColLists.get(tag);
+ cols = Utilities.mergeUniqElems(cols, desc.getCols());
+ prunedColLists.put(tag, cols);
+ }
+ }
+
RowResolver joinRR = cppCtx.getOpToParseCtxMap().get(op).getRR();
RowResolver newJoinRR = new RowResolver();
ArrayList<String> outputCols = new ArrayList<String>();
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java?rev=999637&r1=999636&r2=999637&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/MapJoinProcessor.java Tue Sep 21 21:00:21 2010
@@ -53,6 +53,7 @@ import org.apache.hadoop.hive.ql.lib.Nod
import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx;
import org.apache.hadoop.hive.ql.lib.Rule;
import org.apache.hadoop.hive.ql.lib.RuleRegExp;
+import org.apache.hadoop.hive.ql.parse.ASTNode;
import org.apache.hadoop.hive.ql.parse.ErrorMsg;
import org.apache.hadoop.hive.ql.parse.GenMapRedWalker;
import org.apache.hadoop.hive.ql.parse.OpParseContext;
@@ -60,6 +61,8 @@ import org.apache.hadoop.hive.ql.parse.P
import org.apache.hadoop.hive.ql.parse.QBJoinTree;
import org.apache.hadoop.hive.ql.parse.RowResolver;
import org.apache.hadoop.hive.ql.parse.SemanticException;
+import org.apache.hadoop.hive.ql.parse.TypeCheckCtx;
+import org.apache.hadoop.hive.ql.parse.TypeCheckProcFactory;
import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
import org.apache.hadoop.hive.ql.plan.JoinDesc;
@@ -126,6 +129,8 @@ public class MapJoinProcessor implements
ArrayList<String> outputColumnNames = new ArrayList<String>();
Map<Byte, List<ExprNodeDesc>> keyExprMap = new HashMap<Byte, List<ExprNodeDesc>>();
Map<Byte, List<ExprNodeDesc>> valueExprMap = new HashMap<Byte, List<ExprNodeDesc>>();
+ HashMap<Byte, List<ExprNodeDesc>> filterMap =
+ new HashMap<Byte, List<ExprNodeDesc>>();
// Walk over all the sources (which are guaranteed to be reduce sink
// operators).
@@ -180,6 +185,7 @@ public class MapJoinProcessor implements
newParentOps.get(pos)).getRR();
List<ExprNodeDesc> values = new ArrayList<ExprNodeDesc>();
+ List<ExprNodeDesc> filterDesc = new ArrayList<ExprNodeDesc>();
Iterator<String> keysIter = inputRS.getTableNames().iterator();
while (keysIter.hasNext()) {
@@ -208,7 +214,19 @@ public class MapJoinProcessor implements
}
}
+ TypeCheckCtx tcCtx = new TypeCheckCtx(inputRS);
+ for (ASTNode cond : joinTree.getFilters().get((byte)pos)) {
+
+ ExprNodeDesc filter =
+ (ExprNodeDesc)TypeCheckProcFactory.genExprNode(cond, tcCtx).get(cond);
+ if (filter == null) {
+ throw new SemanticException(tcCtx.getError());
+ }
+ filterDesc.add(filter);
+ }
+
valueExprMap.put(new Byte((byte) pos), values);
+ filterMap.put(new Byte((byte) pos), filterDesc);
}
org.apache.hadoop.hive.ql.plan.JoinCondDesc[] joinCondns = op.getConf()
@@ -247,8 +265,8 @@ public class MapJoinProcessor implements
MapJoinOperator mapJoinOp = (MapJoinOperator) putOpInsertMap(
OperatorFactory.getAndMakeChild(new MapJoinDesc(keyExprMap,
keyTableDesc, valueExprMap, valueTableDescs, outputColumnNames,
- mapJoinPos, joinCondns), new RowSchema(outputRS.getColumnInfos()),
- newPar), outputRS);
+ mapJoinPos, joinCondns, filterMap, op.getConf().getNoOuterJoin()),
+ new RowSchema(outputRS.getColumnInfos()), newPar), outputRS);
mapJoinOp.getConf().setReversedExprs(op.getConf().getReversedExprs());
mapJoinOp.setColumnExprMap(colExprMap);
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/GenMRSkewJoinProcessor.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/GenMRSkewJoinProcessor.java?rev=999637&r1=999636&r2=999637&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/GenMRSkewJoinProcessor.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/GenMRSkewJoinProcessor.java Tue Sep 21 21:00:21 2010
@@ -267,8 +267,8 @@ public final class GenMRSkewJoinProcesso
MapJoinDesc mapJoinDescriptor = new MapJoinDesc(newJoinKeys, keyTblDesc,
newJoinValues, newJoinValueTblDesc, joinDescriptor
- .getOutputColumnNames(), i, joinDescriptor.getConds());
- mapJoinDescriptor.setNoOuterJoin(joinDescriptor.isNoOuterJoin());
+ .getOutputColumnNames(), i, joinDescriptor.getConds(),
+ joinDescriptor.getFilters(), joinDescriptor.getNoOuterJoin());
mapJoinDescriptor.setTagOrder(tags);
mapJoinDescriptor.setHandleSkewJoin(false);
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ErrorMsg.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ErrorMsg.java?rev=999637&r1=999636&r2=999637&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ErrorMsg.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ErrorMsg.java Tue Sep 21 21:00:21 2010
@@ -164,7 +164,9 @@ public enum ErrorMsg {
RESERVED_PART_VAL("Partition value contains a reserved substring"),
HOLD_DDLTIME_ON_NONEXIST_PARTITIONS("HOLD_DDLTIME hint cannot be applied to dynamic " +
"partitions or non-existent partitions"),
- OFFLINE_TABLE_OR_PARTITION("Query against an offline table or partition")
+ OFFLINE_TABLE_OR_PARTITION("Query against an offline table or partition"),
+ OUTERJOIN_USES_FILTERS("The query results could be wrong. " +
+ "Turn on hive.outerjoin.supports.filters"),
;
private String mesg;
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/QBJoinTree.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/QBJoinTree.java?rev=999637&r1=999636&r2=999637&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/QBJoinTree.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/QBJoinTree.java Tue Sep 21 21:00:21 2010
@@ -48,6 +48,9 @@ public class QBJoinTree {
// filters
private ArrayList<ArrayList<ASTNode>> filters;
+ // filters for pushing
+ private ArrayList<ArrayList<ASTNode>> filtersForPushing;
+
// user asked for map-side join
private boolean mapSideJoin;
private List<String> mapAliases;
@@ -172,6 +175,21 @@ public class QBJoinTree {
}
/**
+ * @return the filters for pushing
+ */
+ public ArrayList<ArrayList<ASTNode>> getFiltersForPushing() {
+ return filtersForPushing;
+ }
+
+ /**
+ * @param filters for pushing
+ * the filters to set
+ */
+ public void setFiltersForPushing(ArrayList<ArrayList<ASTNode>> filters) {
+ this.filtersForPushing = filters;
+ }
+
+ /**
* @return the mapSidejoin
*/
public boolean isMapSideJoin() {
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java?rev=999637&r1=999636&r2=999637&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java Tue Sep 21 21:00:21 2010
@@ -1052,6 +1052,7 @@ public class SemanticAnalyzer extends Ba
return;
}
+ JoinType type = joinTree.getJoinCond()[0].getJoinType();
switch (joinCond.getToken().getType()) {
case HiveParser.KW_OR:
throw new SemanticException(ErrorMsg.INVALID_JOIN_CONDITION_3
@@ -1076,6 +1077,11 @@ public class SemanticAnalyzer extends Ba
rightCondAl2, null);
// is it a filter or a join condition
+ // if it is filter see if it can be pushed above the join
+ // filter cannot be pushed if
+ // * join is full outer or
+ // * join is left outer and filter is on left alias or
+ // * join is right outer and filter is on right alias
if (((leftCondAl1.size() != 0) && (leftCondAl2.size() != 0))
|| ((rightCondAl1.size() != 0) && (rightCondAl2.size() != 0))) {
throw new SemanticException(ErrorMsg.INVALID_JOIN_CONDITION_1
@@ -1085,7 +1091,17 @@ public class SemanticAnalyzer extends Ba
if (leftCondAl1.size() != 0) {
if ((rightCondAl1.size() != 0)
|| ((rightCondAl1.size() == 0) && (rightCondAl2.size() == 0))) {
- joinTree.getFilters().get(0).add(joinCond);
+ if (type.equals(JoinType.LEFTOUTER) ||
+ type.equals(JoinType.FULLOUTER)) {
+ if (conf.getBoolVar(HiveConf.ConfVars.HIVEOUTERJOINSUPPORTSFILTERS)) {
+ joinTree.getFilters().get(0).add(joinCond);
+ } else {
+ LOG.warn(ErrorMsg.OUTERJOIN_USES_FILTERS);
+ joinTree.getFiltersForPushing().get(0).add(joinCond);
+ }
+ } else {
+ joinTree.getFiltersForPushing().get(0).add(joinCond);
+ }
} else if (rightCondAl2.size() != 0) {
populateAliases(leftCondAl1, leftCondAl2, leftCondn, joinTree,
leftSrc);
@@ -1095,7 +1111,17 @@ public class SemanticAnalyzer extends Ba
} else if (leftCondAl2.size() != 0) {
if ((rightCondAl2.size() != 0)
|| ((rightCondAl1.size() == 0) && (rightCondAl2.size() == 0))) {
- joinTree.getFilters().get(1).add(joinCond);
+ if (type.equals(JoinType.RIGHTOUTER)
+ || type.equals(JoinType.FULLOUTER)) {
+ if (conf.getBoolVar(HiveConf.ConfVars.HIVEOUTERJOINSUPPORTSFILTERS)) {
+ joinTree.getFilters().get(1).add(joinCond);
+ } else {
+ LOG.warn(ErrorMsg.OUTERJOIN_USES_FILTERS);
+ joinTree.getFiltersForPushing().get(1).add(joinCond);
+ }
+ } else {
+ joinTree.getFiltersForPushing().get(1).add(joinCond);
+ }
} else if (rightCondAl1.size() != 0) {
populateAliases(leftCondAl1, leftCondAl2, leftCondn, joinTree,
leftSrc);
@@ -1103,9 +1129,29 @@ public class SemanticAnalyzer extends Ba
leftSrc);
}
} else if (rightCondAl1.size() != 0) {
- joinTree.getFilters().get(0).add(joinCond);
+ if (type.equals(JoinType.LEFTOUTER)
+ || type.equals(JoinType.FULLOUTER)) {
+ if (conf.getBoolVar(HiveConf.ConfVars.HIVEOUTERJOINSUPPORTSFILTERS)) {
+ joinTree.getFilters().get(0).add(joinCond);
+ } else {
+ LOG.warn(ErrorMsg.OUTERJOIN_USES_FILTERS);
+ joinTree.getFiltersForPushing().get(0).add(joinCond);
+ }
+ } else {
+ joinTree.getFiltersForPushing().get(0).add(joinCond);
+ }
} else {
- joinTree.getFilters().get(1).add(joinCond);
+ if (type.equals(JoinType.RIGHTOUTER)
+ || type.equals(JoinType.FULLOUTER)) {
+ if (conf.getBoolVar(HiveConf.ConfVars.HIVEOUTERJOINSUPPORTSFILTERS)) {
+ joinTree.getFilters().get(1).add(joinCond);
+ } else {
+ LOG.warn(ErrorMsg.OUTERJOIN_USES_FILTERS);
+ joinTree.getFiltersForPushing().get(1).add(joinCond);
+ }
+ } else {
+ joinTree.getFiltersForPushing().get(1).add(joinCond);
+ }
}
break;
@@ -1154,9 +1200,29 @@ public class SemanticAnalyzer extends Ba
}
if (!leftAliasNull) {
- joinTree.getFilters().get(0).add(joinCond);
+ if (type.equals(JoinType.LEFTOUTER)
+ || type.equals(JoinType.FULLOUTER)) {
+ if (conf.getBoolVar(HiveConf.ConfVars.HIVEOUTERJOINSUPPORTSFILTERS)) {
+ joinTree.getFilters().get(0).add(joinCond);
+ } else {
+ LOG.warn(ErrorMsg.OUTERJOIN_USES_FILTERS);
+ joinTree.getFiltersForPushing().get(0).add(joinCond);
+ }
+ } else {
+ joinTree.getFiltersForPushing().get(0).add(joinCond);
+ }
} else {
- joinTree.getFilters().get(1).add(joinCond);
+ if (type.equals(JoinType.RIGHTOUTER)
+ || type.equals(JoinType.FULLOUTER)) {
+ if (conf.getBoolVar(HiveConf.ConfVars.HIVEOUTERJOINSUPPORTSFILTERS)) {
+ joinTree.getFilters().get(1).add(joinCond);
+ } else {
+ LOG.warn(ErrorMsg.OUTERJOIN_USES_FILTERS);
+ joinTree.getFiltersForPushing().get(1).add(joinCond);
+ }
+ } else {
+ joinTree.getFiltersForPushing().get(1).add(joinCond);
+ }
}
break;
@@ -4089,6 +4155,8 @@ public class SemanticAnalyzer extends Ba
HashMap<Byte, List<ExprNodeDesc>> exprMap = new HashMap<Byte, List<ExprNodeDesc>>();
Map<String, ExprNodeDesc> colExprMap = new HashMap<String, ExprNodeDesc>();
HashMap<Integer, Set<String>> posToAliasMap = new HashMap<Integer, Set<String>>();
+ HashMap<Byte, List<ExprNodeDesc>> filterMap =
+ new HashMap<Byte, List<ExprNodeDesc>>();
for (int pos = 0; pos < right.length; ++pos) {
@@ -4098,6 +4166,7 @@ public class SemanticAnalyzer extends Ba
}
ArrayList<ExprNodeDesc> keyDesc = new ArrayList<ExprNodeDesc>();
+ ArrayList<ExprNodeDesc> filterDesc = new ArrayList<ExprNodeDesc>();
Byte tag = Byte.valueOf((byte) (((ReduceSinkDesc) (input.getConf()))
.getTag()));
@@ -4135,8 +4204,12 @@ public class SemanticAnalyzer extends Ba
}
}
}
+ for (ASTNode cond : join.getFilters().get(tag)) {
+ filterDesc.add(genExprNodeDesc(cond, inputRS));
+ }
}
exprMap.put(tag, keyDesc);
+ filterMap.put(tag, filterDesc);
rightOps[pos] = input;
}
@@ -4146,7 +4219,8 @@ public class SemanticAnalyzer extends Ba
joinCondns[i] = new JoinCondDesc(condn);
}
- JoinDesc desc = new JoinDesc(exprMap, outputColumnNames, joinCondns);
+ JoinDesc desc = new JoinDesc(exprMap, outputColumnNames,
+ join.getNoOuterJoin(), joinCondns, filterMap);
desc.setReversedExprs(reversedExprs);
JoinOperator joinOp = (JoinOperator) OperatorFactory.getAndMakeChild(desc,
new RowSchema(outputRS.getColumnInfos()), rightOps);
@@ -4226,7 +4300,7 @@ public class SemanticAnalyzer extends Ba
Operator joinSrcOp = null;
if (leftChild != null) {
Operator joinOp = genJoinOperator(qb, leftChild, map);
- ArrayList<ASTNode> filter = joinTree.getFilters().get(0);
+ ArrayList<ASTNode> filter = joinTree.getFiltersForPushing().get(0);
for (ASTNode cond : filter) {
joinOp = genFilterPlan(qb, cond, joinOp);
}
@@ -4430,11 +4504,10 @@ public class SemanticAnalyzer extends Ba
*/
private void pushJoinFilters(QB qb, QBJoinTree joinTree,
HashMap<String, Operator> map) throws SemanticException {
- ArrayList<ArrayList<ASTNode>> filters = joinTree.getFilters();
if (joinTree.getJoinSrc() != null) {
pushJoinFilters(qb, joinTree.getJoinSrc(), map);
}
-
+ ArrayList<ArrayList<ASTNode>> filters = joinTree.getFiltersForPushing();
int pos = 0;
for (String src : joinTree.getBaseSrc()) {
if (src != null) {
@@ -4477,6 +4550,7 @@ public class SemanticAnalyzer extends Ba
joinTree.setExpressions(new ArrayList<ArrayList<ASTNode>>());
joinTree.setFilters(new ArrayList<ArrayList<ASTNode>>());
+ joinTree.setFiltersForPushing(new ArrayList<ArrayList<ASTNode>>());
// Create joinTree structures to fill them up later
ArrayList<String> rightAliases = new ArrayList<String>();
@@ -4523,6 +4597,7 @@ public class SemanticAnalyzer extends Ba
ArrayList<ASTNode> expressions = new ArrayList<ASTNode>();
ArrayList<ASTNode> filt = new ArrayList<ASTNode>();
+ ArrayList<ASTNode> filters = new ArrayList<ASTNode>();
for (Node exp : child.getChildren()) {
expressions.add((ASTNode) exp);
@@ -4530,6 +4605,7 @@ public class SemanticAnalyzer extends Ba
joinTree.getExpressions().add(expressions);
joinTree.getFilters().add(filt);
+ joinTree.getFiltersForPushing().add(filters);
break;
case HiveParser.KW_PRESERVE:
@@ -4655,6 +4731,12 @@ public class SemanticAnalyzer extends Ba
filters.add(new ArrayList<ASTNode>());
joinTree.setFilters(filters);
+ ArrayList<ArrayList<ASTNode>> filtersForPushing =
+ new ArrayList<ArrayList<ASTNode>>();
+ filtersForPushing.add(new ArrayList<ASTNode>());
+ filtersForPushing.add(new ArrayList<ASTNode>());
+ joinTree.setFiltersForPushing(filtersForPushing);
+
ASTNode joinCond = (ASTNode) joinParseTree.getChild(2);
ArrayList<String> leftSrc = new ArrayList<String>();
parseJoinCondition(joinTree, joinCond, leftSrc);
@@ -4749,14 +4831,19 @@ public class SemanticAnalyzer extends Ba
expr.add(node.getExpressions().get(i + 1));
}
- ArrayList<ArrayList<ASTNode>> filter = target.getFilters();
+ ArrayList<ArrayList<ASTNode>> filters = target.getFilters();
for (int i = 0; i < nodeRightAliases.length; i++) {
- filter.add(node.getFilters().get(i + 1));
+ filters.add(node.getFilters().get(i + 1));
}
- if (node.getFilters().get(0).size() != 0) {
+ ArrayList<ArrayList<ASTNode>> filter = target.getFiltersForPushing();
+ for (int i = 0; i < nodeRightAliases.length; i++) {
+ filter.add(node.getFiltersForPushing().get(i + 1));
+ }
+
+ if (node.getFiltersForPushing().get(0).size() != 0) {
ArrayList<ASTNode> filterPos = filter.get(pos);
- filterPos.addAll(node.getFilters().get(0));
+ filterPos.addAll(node.getFiltersForPushing().get(0));
}
if (qb.getQbJoinTree() == node) {
@@ -6379,40 +6466,8 @@ public class SemanticAnalyzer extends Ba
TypeCheckCtx tcCtx = new TypeCheckCtx(input);
tcCtx.setUnparseTranslator(unparseTranslator);
- // create a walker which walks the tree in a DFS manner while maintaining
- // the operator stack. The dispatcher
- // generates the plan from the operator tree
- Map<Rule, NodeProcessor> opRules = new LinkedHashMap<Rule, NodeProcessor>();
-
- opRules.put(new RuleRegExp("R1", HiveParser.TOK_NULL + "%"),
- TypeCheckProcFactory.getNullExprProcessor());
- opRules.put(new RuleRegExp("R2", HiveParser.Number + "%"),
- TypeCheckProcFactory.getNumExprProcessor());
- opRules
- .put(new RuleRegExp("R3", HiveParser.Identifier + "%|"
- + HiveParser.StringLiteral + "%|" + HiveParser.TOK_CHARSETLITERAL
- + "%|" + HiveParser.KW_IF + "%|" + HiveParser.KW_CASE + "%|"
- + HiveParser.KW_WHEN + "%|" + HiveParser.KW_IN + "%|"
- + HiveParser.KW_ARRAY + "%|" + HiveParser.KW_MAP + "%|"
- + HiveParser.KW_STRUCT + "%"), TypeCheckProcFactory
- .getStrExprProcessor());
- opRules.put(new RuleRegExp("R4", HiveParser.KW_TRUE + "%|"
- + HiveParser.KW_FALSE + "%"), TypeCheckProcFactory
- .getBoolExprProcessor());
- opRules.put(new RuleRegExp("R5", HiveParser.TOK_TABLE_OR_COL + "%"),
- TypeCheckProcFactory.getColumnExprProcessor());
-
- // The dispatcher fires the processor corresponding to the closest matching
- // rule and passes the context along
- Dispatcher disp = new DefaultRuleDispatcher(TypeCheckProcFactory
- .getDefaultExprProcessor(), opRules, tcCtx);
- GraphWalker ogw = new DefaultGraphWalker(disp);
-
- // Create a list of topop nodes
- ArrayList<Node> topNodes = new ArrayList<Node>();
- topNodes.add(expr);
- HashMap<Node, Object> nodeOutputs = new HashMap<Node, Object>();
- ogw.startWalking(topNodes, nodeOutputs);
+ HashMap<Node, Object> nodeOutputs =
+ TypeCheckProcFactory.genExprNode(expr, tcCtx);
ExprNodeDesc desc = (ExprNodeDesc) nodeOutputs.get(expr);
if (desc == null) {
throw new SemanticException(tcCtx.getError());
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java?rev=999637&r1=999636&r2=999637&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java Tue Sep 21 21:00:21 2010
@@ -21,21 +21,28 @@ package org.apache.hadoop.hive.ql.parse;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
+import java.util.LinkedHashMap;
import java.util.List;
+import java.util.Map;
import java.util.Stack;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hive.ql.exec.AmbiguousMethodException;
import org.apache.hadoop.hive.ql.exec.ColumnInfo;
import org.apache.hadoop.hive.ql.exec.FunctionInfo;
import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
import org.apache.hadoop.hive.ql.exec.UDFArgumentLengthException;
import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException;
+import org.apache.hadoop.hive.ql.lib.DefaultGraphWalker;
+import org.apache.hadoop.hive.ql.lib.DefaultRuleDispatcher;
+import org.apache.hadoop.hive.ql.lib.Dispatcher;
+import org.apache.hadoop.hive.ql.lib.GraphWalker;
import org.apache.hadoop.hive.ql.lib.Node;
import org.apache.hadoop.hive.ql.lib.NodeProcessor;
import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx;
+import org.apache.hadoop.hive.ql.lib.Rule;
+import org.apache.hadoop.hive.ql.lib.RuleRegExp;
import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
@@ -111,6 +118,46 @@ public final class TypeCheckProcFactory
return desc;
}
+ public static HashMap<Node, Object> genExprNode(ASTNode expr,
+ TypeCheckCtx tcCtx) throws SemanticException {
+ // Create the walker, the rules dispatcher and the context.
+ // create a walker which walks the tree in a DFS manner while maintaining
+ // the operator stack. The dispatcher
+ // generates the plan from the operator tree
+ Map<Rule, NodeProcessor> opRules = new LinkedHashMap<Rule, NodeProcessor>();
+
+ opRules.put(new RuleRegExp("R1", HiveParser.TOK_NULL + "%"),
+ getNullExprProcessor());
+ opRules.put(new RuleRegExp("R2", HiveParser.Number + "%"),
+ getNumExprProcessor());
+ opRules
+ .put(new RuleRegExp("R3", HiveParser.Identifier + "%|"
+ + HiveParser.StringLiteral + "%|" + HiveParser.TOK_CHARSETLITERAL
+ + "%|" + HiveParser.KW_IF + "%|" + HiveParser.KW_CASE + "%|"
+ + HiveParser.KW_WHEN + "%|" + HiveParser.KW_IN + "%|"
+ + HiveParser.KW_ARRAY + "%|" + HiveParser.KW_MAP + "%|"
+ + HiveParser.KW_STRUCT + "%"),
+ getStrExprProcessor());
+ opRules.put(new RuleRegExp("R4", HiveParser.KW_TRUE + "%|"
+ + HiveParser.KW_FALSE + "%"), getBoolExprProcessor());
+ opRules.put(new RuleRegExp("R5", HiveParser.TOK_TABLE_OR_COL + "%"),
+ getColumnExprProcessor());
+
+ // The dispatcher fires the processor corresponding to the closest matching
+ // rule and passes the context along
+ Dispatcher disp = new DefaultRuleDispatcher(getDefaultExprProcessor(),
+ opRules, tcCtx);
+ GraphWalker ogw = new DefaultGraphWalker(disp);
+
+ // Create a list of topop nodes
+ ArrayList<Node> topNodes = new ArrayList<Node>();
+ topNodes.add(expr);
+ HashMap<Node, Object> nodeOutputs = new HashMap<Node, Object>();
+ ogw.startWalking(topNodes, nodeOutputs);
+
+ return nodeOutputs;
+ }
+
/**
* Processor for processing NULL expression.
*/
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/JoinDesc.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/JoinDesc.java?rev=999637&r1=999636&r2=999637&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/JoinDesc.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/JoinDesc.java Tue Sep 21 21:00:21 2010
@@ -48,6 +48,9 @@ public class JoinDesc implements Seriali
// alias to key mapping
private Map<Byte, List<ExprNodeDesc>> exprs;
+ // alias to filter mapping
+ private Map<Byte, List<ExprNodeDesc>> filters;
+
// used for create joinOutputObjectInspector
protected List<String> outputColumnNames;
@@ -67,11 +70,12 @@ public class JoinDesc implements Seriali
public JoinDesc(final Map<Byte, List<ExprNodeDesc>> exprs,
List<String> outputColumnNames, final boolean noOuterJoin,
- final JoinCondDesc[] conds) {
+ final JoinCondDesc[] conds, final Map<Byte, List<ExprNodeDesc>> filters) {
this.exprs = exprs;
this.outputColumnNames = outputColumnNames;
this.noOuterJoin = noOuterJoin;
this.conds = conds;
+ this.filters = filters;
tagOrder = new Byte[exprs.size()];
for (int i = 0; i < tagOrder.length; i++) {
@@ -80,13 +84,19 @@ public class JoinDesc implements Seriali
}
public JoinDesc(final Map<Byte, List<ExprNodeDesc>> exprs,
+ List<String> outputColumnNames, final boolean noOuterJoin,
+ final JoinCondDesc[] conds) {
+ this(exprs, outputColumnNames, noOuterJoin, conds, null);
+ }
+
+ public JoinDesc(final Map<Byte, List<ExprNodeDesc>> exprs,
List<String> outputColumnNames) {
this(exprs, outputColumnNames, true, null);
}
public JoinDesc(final Map<Byte, List<ExprNodeDesc>> exprs,
List<String> outputColumnNames, final JoinCondDesc[] conds) {
- this(exprs, outputColumnNames, false, conds);
+ this(exprs, outputColumnNames, true, conds, null);
}
public JoinDesc(JoinDesc clone) {
@@ -102,6 +112,7 @@ public class JoinDesc implements Seriali
this.skewKeysValuesTables = clone.skewKeysValuesTables;
this.smallKeysDirMap = clone.smallKeysDirMap;
this.tagOrder = clone.tagOrder;
+ this.filters = clone.filters;
}
public Map<Byte, List<ExprNodeDesc>> getExprs() {
@@ -149,6 +160,59 @@ public class JoinDesc implements Seriali
this.exprs = exprs;
}
+ /**
+ * Get the string representation of filters.
+ *
+ * Returns null if they are no filters.
+ *
+ * @return Map from alias to filters on the alias.
+ */
+ @Explain(displayName = "filter predicates")
+ public Map<Byte, String> getFiltersStringMap() {
+ if (getFilters() == null || getFilters().size() == 0) {
+ return null;
+ }
+
+ LinkedHashMap<Byte, String> ret = new LinkedHashMap<Byte, String>();
+ boolean filtersPresent = false;
+
+ for (Map.Entry<Byte, List<ExprNodeDesc>> ent : getFilters().entrySet()) {
+ StringBuilder sb = new StringBuilder();
+ boolean first = true;
+ if (ent.getValue() != null) {
+ if (ent.getValue().size() != 0) {
+ filtersPresent = true;
+ }
+ for (ExprNodeDesc expr : ent.getValue()) {
+ if (!first) {
+ sb.append(" ");
+ }
+
+ first = false;
+ sb.append("{");
+ sb.append(expr.getExprString());
+ sb.append("}");
+ }
+ }
+ ret.put(ent.getKey(), sb.toString());
+ }
+
+ if (filtersPresent) {
+ return ret;
+ } else {
+ return null;
+ }
+ }
+
+
+ public Map<Byte, List<ExprNodeDesc>> getFilters() {
+ return filters;
+ }
+
+ public void setFilters(Map<Byte, List<ExprNodeDesc>> filters) {
+ this.filters = filters;
+ }
+
@Explain(displayName = "outputColumnNames")
public List<String> getOutputColumnNames() {
return outputColumnNames;
@@ -288,14 +352,7 @@ public class JoinDesc implements Seriali
}
public boolean isNoOuterJoin() {
- for (org.apache.hadoop.hive.ql.plan.JoinCondDesc cond : conds) {
- if (cond.getType() == JoinDesc.FULL_OUTER_JOIN
- || (cond.getType() == JoinDesc.LEFT_OUTER_JOIN)
- || cond.getType() == JoinDesc.RIGHT_OUTER_JOIN) {
- return false;
- }
- }
- return true;
+ return noOuterJoin;
}
public void setKeyTableDesc(TableDesc keyTblDesc) {
Modified: hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/MapJoinDesc.java
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/MapJoinDesc.java?rev=999637&r1=999636&r2=999637&view=diff
==============================================================================
--- hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/MapJoinDesc.java (original)
+++ hadoop/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/MapJoinDesc.java Tue Sep 21 21:00:21 2010
@@ -68,8 +68,9 @@ public class MapJoinDesc extends JoinDes
public MapJoinDesc(final Map<Byte, List<ExprNodeDesc>> keys,
final TableDesc keyTblDesc, final Map<Byte, List<ExprNodeDesc>> values,
final List<TableDesc> valueTblDescs, List<String> outputColumnNames,
- final int posBigTable, final JoinCondDesc[] conds) {
- super(values, outputColumnNames, conds);
+ final int posBigTable, final JoinCondDesc[] conds,
+ final Map<Byte, List<ExprNodeDesc>> filters, boolean noOuterJoin) {
+ super(values, outputColumnNames, noOuterJoin, conds, filters);
this.keys = keys;
this.keyTblDesc = keyTblDesc;
this.valueTblDescs = valueTblDescs;
Added: hadoop/hive/trunk/ql/src/test/queries/clientpositive/join_filters.q
URL: http://svn.apache.org/viewvc/hadoop/hive/trunk/ql/src/test/queries/clientpositive/join_filters.q?rev=999637&view=auto
==============================================================================
--- hadoop/hive/trunk/ql/src/test/queries/clientpositive/join_filters.q (added)
+++ hadoop/hive/trunk/ql/src/test/queries/clientpositive/join_filters.q Tue Sep 21 21:00:21 2010
@@ -0,0 +1,155 @@
+CREATE TABLE myinput1(key int, value int);
+LOAD DATA LOCAL INPATH '../data/files/in3.txt' INTO TABLE myinput1;
+
+SELECT * FROM myinput1 a JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+SELECT * FROM myinput1 a LEFT OUTER JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+SELECT * FROM myinput1 a RIGHT OUTER JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+SELECT * FROM myinput1 a FULL OUTER JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+
+SELECT * FROM myinput1 a JOIN myinput1 b ON a.key = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+SELECT * FROM myinput1 a JOIN myinput1 b ON a.key = b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+SELECT * FROM myinput1 a JOIN myinput1 b ON a.value = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+SELECT * FROM myinput1 a JOIN myinput1 b ON a.value = b.value and a.key=b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+
+SELECT * FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.key = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+SELECT * FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.value = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+SELECT * FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.key = b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+SELECT * FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.key = b.key and a.value=b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+
+SELECT * FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.key = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+SELECT * FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.key = b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+SELECT * FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.value = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+SELECT * FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.key=b.key and a.value = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+
+SELECT * FROM myinput1 a FULL OUTER JOIN myinput1 b ON a.key = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+SELECT * FROM myinput1 a FULL OUTER JOIN myinput1 b ON a.key = b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+SELECT * FROM myinput1 a FULL OUTER JOIN myinput1 b ON a.value = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+SELECT * FROM myinput1 a FULL OUTER JOIN myinput1 b ON a.value = b.value and a.key=b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+
+SELECT * from myinput1 a LEFT OUTER JOIN myinput1 b ON (a.value=b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value) RIGHT OUTER JOIN myinput1 c ON (b.value=c.value AND c.key > 40 AND c.value > 50 AND c.key = c.value AND b.key > 40 AND b.value > 50 AND b.key = b.value);
+SELECT * from myinput1 a RIGHT OUTER JOIN myinput1 b ON (a.value=b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value) LEFT OUTER JOIN myinput1 c ON (b.value=c.value AND c.key > 40 AND c.value > 50 AND c.key = c.value AND b.key > 40 AND b.value > 50 AND b.key = b.value);
+SELECT * FROM myinput1 a LEFT OUTER JOIN myinput1 b RIGHT OUTER JOIN myinput1 c ON a.value = b.value and b.value = c.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value AND c.key > 40 AND c.value > 50 AND c.key = c.value;
+SELECT * from myinput1 a LEFT OUTER JOIN myinput1 b ON (a.value=b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value) RIGHT OUTER JOIN myinput1 c ON (b.key=c.key AND c.key > 40 AND c.value > 50 AND c.key = c.value AND b.key > 40 AND b.value > 50 AND b.key = b.value);
+SELECT * from myinput1 a RIGHT OUTER JOIN myinput1 b ON (a.value=b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value) LEFT OUTER JOIN myinput1 c ON (b.key=c.key AND c.key > 40 AND c.value > 50 AND c.key = c.value AND b.key > 40 AND b.value > 50 AND b.key = b.value);
+SELECT * FROM myinput1 a LEFT OUTER JOIN myinput1 b RIGHT OUTER JOIN myinput1 c ON a.value = b.value and b.key = c.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value AND c.key > 40 AND c.value > 50 AND c.key = c.value;
+
+SELECT /*+ MAPJOIN(a) */ * FROM myinput1 a JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+SELECT /*+ MAPJOIN(a) */ * FROM myinput1 a JOIN myinput1 b ON a.key = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+SELECT /*+ MAPJOIN(a) */ * FROM myinput1 a JOIN myinput1 b ON a.key = b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+SELECT /*+ MAPJOIN(a) */ * FROM myinput1 a JOIN myinput1 b ON a.value = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+SELECT /*+ MAPJOIN(a) */ * FROM myinput1 a JOIN myinput1 b ON a.value = b.value and a.key = b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+
+SELECT /*+ MAPJOIN(b) */ * FROM myinput1 a JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+SELECT /*+ MAPJOIN(b) */ * FROM myinput1 a JOIN myinput1 b ON a.key = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+SELECT /*+ MAPJOIN(b) */ * FROM myinput1 a JOIN myinput1 b ON a.key = b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+SELECT /*+ MAPJOIN(b) */ * FROM myinput1 a JOIN myinput1 b ON a.value = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+SELECT /*+ MAPJOIN(b) */ * FROM myinput1 a JOIN myinput1 b ON a.value = b.value and a.key = b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+
+SELECT /*+ MAPJOIN(b) */ * FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.key = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+SELECT /*+ MAPJOIN(b) */ * FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.key = b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+SELECT /*+ MAPJOIN(b) */ * FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.value = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+
+SELECT /*+ MAPJOIN(a) */ * FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.key = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+SELECT /*+ MAPJOIN(a) */ * FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.key = b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+SELECT /*+ MAPJOIN(a) */ * FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.value = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+
+CREATE TABLE smb_input1(key int, value int) CLUSTERED BY (key) SORTED BY (key) INTO 2 BUCKETS;
+CREATE TABLE smb_input2(key int, value int) CLUSTERED BY (value) SORTED BY (value) INTO 2 BUCKETS;
+LOAD DATA LOCAL INPATH '../data/files/in1.txt' into table smb_input1;
+LOAD DATA LOCAL INPATH '../data/files/in2.txt' into table smb_input1;
+LOAD DATA LOCAL INPATH '../data/files/in1.txt' into table smb_input2;
+LOAD DATA LOCAL INPATH '../data/files/in2.txt' into table smb_input2;
+
+SET hive.optimize.bucketmapjoin = true;
+SET hive.optimize.bucketmapjoin.sortedmerge = true;
+SET hive.input.format = org.apache.hadoop.hive.ql.io.BucketizedHiveInputFormat;
+
+SELECT /*+ MAPJOIN(a) */ * FROM smb_input1 a JOIN smb_input1 b ON a.key = b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value ORDER BY a.key;
+SELECT /*+ MAPJOIN(a) */ * FROM smb_input1 a JOIN smb_input2 b ON a.key = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value ORDER BY a.key;
+SELECT /*+ MAPJOIN(a) */ * FROM smb_input2 a JOIN smb_input2 b ON a.value = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value ORDER BY a.value;
+SELECT /*+ MAPJOIN(a) */ * FROM smb_input1 a JOIN smb_input1 b ON a.key = b.key AND a.value = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value ORDER BY a.key;
+
+SELECT /*+ MAPJOIN(b) */ * FROM smb_input1 a JOIN smb_input1 b ON a.key = b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value ORDER BY a.key;
+SELECT /*+ MAPJOIN(b) */ * FROM smb_input1 a JOIN smb_input2 b ON a.key = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value ORDER BY a.key;
+SELECT /*+ MAPJOIN(b) */ * FROM smb_input2 a JOIN smb_input2 b ON a.value = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value ORDER BY a.value;
+SELECT /*+ MAPJOIN(b) */ * FROM smb_input2 a JOIN smb_input2 b ON a.key = b.key AND a.value = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value ORDER BY a.key;
+
+SELECT /*+ MAPJOIN(b) */ * FROM smb_input1 a LEFT OUTER JOIN smb_input1 b ON a.key = b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value ORDER BY a.key, a.value;
+SELECT /*+ MAPJOIN(b) */ * FROM smb_input1 a LEFT OUTER JOIN smb_input2 b ON a.key = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value ORDER BY a.key, a.value;
+SELECT /*+ MAPJOIN(b) */ * FROM smb_input2 a LEFT OUTER JOIN smb_input2 b ON a.value = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value ORDER BY a.key, a.value;
+
+SELECT /*+ MAPJOIN(a) */ * FROM smb_input1 a RIGHT OUTER JOIN smb_input1 b ON a.key = b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value ORDER BY b.key, b.value;
+SELECT /*+ MAPJOIN(a) */ * FROM smb_input1 a RIGHT OUTER JOIN smb_input2 b ON a.key = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value ORDER BY b.key, b.value;
+SELECT /*+ MAPJOIN(a) */ * FROM smb_input2 a RIGHT OUTER JOIN smb_input2 b ON a.value = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value ORDER BY b.key, b.value;
+
+SET hive.outerjoin.supports.filters = false;
+
+SELECT * FROM myinput1 a JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+SELECT * FROM myinput1 a LEFT OUTER JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+SELECT * FROM myinput1 a RIGHT OUTER JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+SELECT * FROM myinput1 a FULL OUTER JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+
+SELECT * FROM myinput1 a JOIN myinput1 b ON a.key = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+SELECT * FROM myinput1 a JOIN myinput1 b ON a.key = b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+SELECT * FROM myinput1 a JOIN myinput1 b ON a.value = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+SELECT * FROM myinput1 a JOIN myinput1 b ON a.value = b.value and a.key=b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+
+SELECT * FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.key = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+SELECT * FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.value = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+SELECT * FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.key = b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+SELECT * FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.key = b.key and a.value=b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+
+SELECT * FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.key = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+SELECT * FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.key = b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+SELECT * FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.value = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+SELECT * FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.key=b.key and a.value = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+
+SELECT * FROM myinput1 a FULL OUTER JOIN myinput1 b ON a.key = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+SELECT * FROM myinput1 a FULL OUTER JOIN myinput1 b ON a.key = b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+SELECT * FROM myinput1 a FULL OUTER JOIN myinput1 b ON a.value = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+SELECT * FROM myinput1 a FULL OUTER JOIN myinput1 b ON a.value = b.value and a.key=b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+
+SELECT * from myinput1 a LEFT OUTER JOIN myinput1 b ON (a.value=b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value) RIGHT OUTER JOIN myinput1 c ON (b.value=c.value AND c.key > 40 AND c.value > 50 AND c.key = c.value AND b.key > 40 AND b.value > 50 AND b.key = b.value);
+SELECT * from myinput1 a RIGHT OUTER JOIN myinput1 b ON (a.value=b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value) LEFT OUTER JOIN myinput1 c ON (b.value=c.value AND c.key > 40 AND c.value > 50 AND c.key = c.value AND b.key > 40 AND b.value > 50 AND b.key = b.value);
+SELECT * FROM myinput1 a LEFT OUTER JOIN myinput1 b RIGHT OUTER JOIN myinput1 c ON a.value = b.value and b.value = c.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value AND c.key > 40 AND c.value > 50 AND c.key = c.value;
+SELECT * from myinput1 a LEFT OUTER JOIN myinput1 b ON (a.value=b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value) RIGHT OUTER JOIN myinput1 c ON (b.key=c.key AND c.key > 40 AND c.value > 50 AND c.key = c.value AND b.key > 40 AND b.value > 50 AND b.key = b.value);
+SELECT * from myinput1 a RIGHT OUTER JOIN myinput1 b ON (a.value=b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value) LEFT OUTER JOIN myinput1 c ON (b.key=c.key AND c.key > 40 AND c.value > 50 AND c.key = c.value AND b.key > 40 AND b.value > 50 AND b.key = b.value);
+SELECT * FROM myinput1 a LEFT OUTER JOIN myinput1 b RIGHT OUTER JOIN myinput1 c ON a.value = b.value and b.key = c.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value AND c.key > 40 AND c.value > 50 AND c.key = c.value;
+
+SELECT /*+ MAPJOIN(a) */ * FROM myinput1 a JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+SELECT /*+ MAPJOIN(a) */ * FROM myinput1 a JOIN myinput1 b ON a.key = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+SELECT /*+ MAPJOIN(a) */ * FROM myinput1 a JOIN myinput1 b ON a.key = b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+SELECT /*+ MAPJOIN(a) */ * FROM myinput1 a JOIN myinput1 b ON a.value = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+SELECT /*+ MAPJOIN(a) */ * FROM myinput1 a JOIN myinput1 b ON a.value = b.value and a.key = b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+
+SELECT /*+ MAPJOIN(b) */ * FROM myinput1 a JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+SELECT /*+ MAPJOIN(b) */ * FROM myinput1 a JOIN myinput1 b ON a.key = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+SELECT /*+ MAPJOIN(b) */ * FROM myinput1 a JOIN myinput1 b ON a.key = b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+SELECT /*+ MAPJOIN(b) */ * FROM myinput1 a JOIN myinput1 b ON a.value = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+SELECT /*+ MAPJOIN(b) */ * FROM myinput1 a JOIN myinput1 b ON a.value = b.value and a.key = b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+
+SELECT /*+ MAPJOIN(b) */ * FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.key = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+SELECT /*+ MAPJOIN(b) */ * FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.key = b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+SELECT /*+ MAPJOIN(b) */ * FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.value = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+
+SELECT /*+ MAPJOIN(a) */ * FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.key = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+SELECT /*+ MAPJOIN(a) */ * FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.key = b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+SELECT /*+ MAPJOIN(a) */ * FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.value = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+
+SELECT /*+ MAPJOIN(a) */ * FROM smb_input1 a JOIN smb_input1 b ON a.key = b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value ORDER BY a.key;
+SELECT /*+ MAPJOIN(a) */ * FROM smb_input1 a JOIN smb_input2 b ON a.key = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value ORDER BY a.key;
+SELECT /*+ MAPJOIN(a) */ * FROM smb_input2 a JOIN smb_input2 b ON a.value = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value ORDER BY a.value;
+SELECT /*+ MAPJOIN(a) */ * FROM smb_input1 a JOIN smb_input1 b ON a.key = b.key AND a.value = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value ORDER BY a.key;
+
+SELECT /*+ MAPJOIN(b) */ * FROM smb_input1 a JOIN smb_input1 b ON a.key = b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value ORDER BY a.key;
+SELECT /*+ MAPJOIN(b) */ * FROM smb_input1 a JOIN smb_input2 b ON a.key = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value ORDER BY a.key;
+SELECT /*+ MAPJOIN(b) */ * FROM smb_input2 a JOIN smb_input2 b ON a.value = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value ORDER BY a.value;
+SELECT /*+ MAPJOIN(b) */ * FROM smb_input2 a JOIN smb_input2 b ON a.key = b.key AND a.value = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value ORDER BY a.key;
+
+SELECT /*+ MAPJOIN(b) */ * FROM smb_input1 a LEFT OUTER JOIN smb_input1 b ON a.key = b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value ORDER BY a.key, a.value;
+SELECT /*+ MAPJOIN(b) */ * FROM smb_input1 a LEFT OUTER JOIN smb_input2 b ON a.key = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value ORDER BY a.key, a.value;
+SELECT /*+ MAPJOIN(b) */ * FROM smb_input2 a LEFT OUTER JOIN smb_input2 b ON a.value = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value ORDER BY a.key, a.value;
+
+SELECT /*+ MAPJOIN(a) */ * FROM smb_input1 a RIGHT OUTER JOIN smb_input1 b ON a.key = b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value ORDER BY b.key, b.value;
+SELECT /*+ MAPJOIN(a) */ * FROM smb_input1 a RIGHT OUTER JOIN smb_input2 b ON a.key = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value ORDER BY b.key, b.value;
+SELECT /*+ MAPJOIN(a) */ * FROM smb_input2 a RIGHT OUTER JOIN smb_input2 b ON a.value = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value ORDER BY b.key, b.value;